You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
122 lines
3.8 KiB
JavaScript
122 lines
3.8 KiB
JavaScript
"use strict";
|
|
|
|
const assert = require("assert");
|
|
|
|
const assureResponse = require("../../shared/assure-response");
|
|
const createDatasheet = require("../../shared/create-datasheet");
|
|
|
|
// LCSC
|
|
// TODO: Validate response formats with validatem instead
|
|
|
|
const PAGE_SIZE = 100; // Maximum amount of items per page that the API allows you to request
|
|
|
|
module.exports = function ({ session }) {
|
|
return {
|
|
seed: [{
|
|
id: "lcsc:home",
|
|
tags: [ "lcsc:home" ],
|
|
data: {}
|
|
}],
|
|
tags: {
|
|
"lcsc:home": [ "lcsc:findCategories" ],
|
|
"lcsc:category": [ "lcsc:scrapeCategory" ],
|
|
"lcsc:product": [ "lcsc:normalizeProduct" ],
|
|
},
|
|
tasks: {
|
|
"lcsc:findCategories": {
|
|
ttl: "30d",
|
|
version: "1",
|
|
run: async function ({ storeItem }) {
|
|
let response = await session.get("https://wmsc.lcsc.com/wmsc/product/catalogs/search");
|
|
|
|
assureResponse(response);
|
|
assert(response.body.result.length > 0);
|
|
assert(response.body.code === 200);
|
|
|
|
function processCategoryEntries(categories) {
|
|
for (let category of categories) {
|
|
let productCount = category.productNum;
|
|
let pageCount = Math.ceil(productCount / PAGE_SIZE);
|
|
|
|
// NOTE: We do *not* use the page count indicated by the API, but instead calculate it ourself from the product count. This is because the API-specified page count will cap out at the equivalent of 10k items, even when more pages than that are actually available.
|
|
for (let i = 1; i <= pageCount; i++) {
|
|
storeItem({
|
|
id: `lcsc:category:${category.catalogId}:page-${i}`,
|
|
tags: [ "lcsc:category" ],
|
|
data: {
|
|
... category,
|
|
pageNumber: i
|
|
}
|
|
});
|
|
}
|
|
|
|
if (category.childCatelogs != null) {
|
|
processCategoryEntries(category.childCatelogs);
|
|
}
|
|
}
|
|
}
|
|
|
|
processCategoryEntries(response.body.result);
|
|
}
|
|
},
|
|
"lcsc:scrapeCategory": {
|
|
ttl: "30d",
|
|
taskInterval: "1m",
|
|
run: async function ({ data, storeItem, deleteItem }) {
|
|
let response = await session.post(`https://wmsc.lcsc.com/wmsc/product/search/list`, {
|
|
catalogIdList: [ data.catalogId ],
|
|
currentPage: data.pageNumber,
|
|
pageSize: PAGE_SIZE,
|
|
paramNameValueMap: {}
|
|
}, { encodeJSON: true });
|
|
|
|
console.log({ response });
|
|
assureResponse(response);
|
|
assert(response.body.code === 200);
|
|
assert(response.body.result?.dataList != null); // Missing from stale queued requests?
|
|
assert(response.body.result.dataList.length > 0);
|
|
|
|
for (let item of response.body.result.dataList) {
|
|
storeItem({
|
|
// NOTE: item.productId seems like the database ID on the website, but item.productCode is the actual LCSC part number used internally for inventory management, so we use that for identification instead
|
|
id: `lcsc:product:${item.productCode}`,
|
|
tags: [ "lcsc:product" ],
|
|
data: item
|
|
});
|
|
}
|
|
|
|
// We don't keep around page items, because the amount of pages for a category can change, and so this isn't a stable identifier. They'll be recreated on the next category scrape anyway.
|
|
deleteItem();
|
|
}
|
|
},
|
|
"lcsc:normalizeProduct": {
|
|
version: "7",
|
|
parallelTasks: 50,
|
|
run: async function (api) {
|
|
let { data } = api;
|
|
|
|
function clean(url) {
|
|
// LCSC sometimes uses null, and sometimes uses empty string
|
|
if (url == "") {
|
|
return null;
|
|
} else {
|
|
return url;
|
|
}
|
|
}
|
|
|
|
createDatasheet(api, {
|
|
priority: 0.4,
|
|
source: "lcsc",
|
|
manufacturer: data.brandNameEn,
|
|
productID: data.productCode,
|
|
name: data.productModel,
|
|
description: data.productIntroEn,
|
|
// For TI datasheets, LCSC references an external URL using pdfLinkUrl instead of a local copy
|
|
url: clean(data.pdfUrl) ?? clean(data.pdfLinkUrl)
|
|
});
|
|
}
|
|
},
|
|
}
|
|
};
|
|
};
|