You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
109 lines
3.3 KiB
JavaScript
109 lines
3.3 KiB
JavaScript
"use strict";
|
|
|
|
const assert = require("assert");
|
|
|
|
const assureResponse = require("../../shared/assure-response");
|
|
const createDatasheet = require("../../shared/create-datasheet");
|
|
|
|
// LCSC
|
|
// TODO: Validate response formats with validatem instead
|
|
|
|
module.exports = function ({ session }) {
|
|
return {
|
|
seed: [{
|
|
id: "lcsc:home",
|
|
tags: [ "lcsc:home" ],
|
|
data: {}
|
|
}],
|
|
tags: {
|
|
"lcsc:home": [ "lcsc:findCategories" ],
|
|
"lcsc:category": [ "lcsc:scrapeCategory" ],
|
|
"lcsc:product": [ "lcsc:normalizeProduct" ],
|
|
},
|
|
tasks: {
|
|
"lcsc:findCategories": {
|
|
ttl: "30d",
|
|
version: "1",
|
|
run: async function ({ storeItem }) {
|
|
let response = await session.get("https://wwwapi.lcsc.com/v1/home/category");
|
|
|
|
assureResponse(response);
|
|
assert(response.body.length > 0);
|
|
assert(response.statusCode === 200);
|
|
|
|
function processCategoryEntries(categories) {
|
|
for (let category of categories) {
|
|
let productCount = category.productNum;
|
|
let pageCount = Math.ceil(productCount / 500);
|
|
|
|
// NOTE: We do *not* use the page count indicated by the API, but instead calculate it ourself from the product count. This is because the API-specified page count will cap out at the equivalent of 10k items, even when more pages than that are actually available.
|
|
for (let i = 1; i <= pageCount; i++) {
|
|
storeItem({
|
|
id: `lcsc:category:${category.catalogId}:page-${i}`,
|
|
tags: [ "lcsc:category" ],
|
|
data: {
|
|
... category,
|
|
pageNumber: i
|
|
}
|
|
});
|
|
}
|
|
|
|
if (category.childCatelogs != null) {
|
|
processCategoryEntries(category.childCatelogs);
|
|
}
|
|
}
|
|
}
|
|
|
|
processCategoryEntries(response.body);
|
|
}
|
|
},
|
|
"lcsc:scrapeCategory": {
|
|
ttl: "30d",
|
|
taskInterval: "1m",
|
|
run: async function ({ data, storeItem, deleteItem }) {
|
|
let response = await session.post(`https://wwwapi.lcsc.com/v1/products/list`, {
|
|
catalogIdList: [ data.catalogId ],
|
|
currentPage: data.pageNumber,
|
|
pageSize: 500,
|
|
paramNameValueMap: {}
|
|
});
|
|
|
|
assureResponse(response);
|
|
assert(response.statusCode === 200);
|
|
assert(response.body.productList != null); // Missing from stale queued requests?
|
|
assert(response.body.productList.length > 0);
|
|
|
|
for (let item of response.body.productList) {
|
|
storeItem({
|
|
// NOTE: item.productId seems like the database ID on the website, but item.productCode is the actual LCSC part number used internally for inventory management, so we use that for identification instead
|
|
id: `lcsc:product:${item.productCode}`,
|
|
tags: [ "lcsc:product" ],
|
|
data: item
|
|
});
|
|
}
|
|
|
|
// We don't keep around page items, because the amount of pages for a category can change, and so this isn't a stable identifier. They'll be recreated on the next category scrape anyway.
|
|
deleteItem();
|
|
}
|
|
},
|
|
"lcsc:normalizeProduct": {
|
|
version: "7",
|
|
parallelTasks: 50,
|
|
run: async function (api) {
|
|
let { data } = api;
|
|
|
|
createDatasheet(api, {
|
|
priority: 0.4,
|
|
source: "lcsc",
|
|
manufacturer: data.brandNameEn,
|
|
productID: data.productCode,
|
|
name: data.productModel,
|
|
description: data.productIntroEn,
|
|
url: data.pdfUrl
|
|
});
|
|
}
|
|
},
|
|
}
|
|
};
|
|
};
|