lcsc: Update for new site API

master
Sven Slootweg 1 year ago
parent d972746f35
commit f83e706739

@ -8,6 +8,8 @@ const createDatasheet = require("../../shared/create-datasheet");
// LCSC
// TODO: Validate response formats with validatem instead
const PAGE_SIZE = 100; // Maximum amount of items per page that the API allows you to request
module.exports = function ({ session }) {
return {
seed: [{
@ -25,16 +27,16 @@ module.exports = function ({ session }) {
ttl: "30d",
version: "1",
run: async function ({ storeItem }) {
let response = await session.get("https://wwwapi.lcsc.com/v1/home/category");
let response = await session.get("https://wmsc.lcsc.com/wmsc/product/catalogs/search");
assureResponse(response);
assert(response.body.length > 0);
assert(response.statusCode === 200);
assert(response.body.result.length > 0);
assert(response.body.code === 200);
function processCategoryEntries(categories) {
for (let category of categories) {
let productCount = category.productNum;
let pageCount = Math.ceil(productCount / 500);
let pageCount = Math.ceil(productCount / PAGE_SIZE);
// NOTE: We do *not* use the page count indicated by the API, but instead calculate it ourself from the product count. This is because the API-specified page count will cap out at the equivalent of 10k items, even when more pages than that are actually available.
for (let i = 1; i <= pageCount; i++) {
@ -54,26 +56,27 @@ module.exports = function ({ session }) {
}
}
processCategoryEntries(response.body);
processCategoryEntries(response.body.result);
}
},
"lcsc:scrapeCategory": {
ttl: "30d",
taskInterval: "1m",
run: async function ({ data, storeItem, deleteItem }) {
let response = await session.post(`https://wwwapi.lcsc.com/v1/products/list`, {
let response = await session.post(`https://wmsc.lcsc.com/wmsc/product/search/list`, {
catalogIdList: [ data.catalogId ],
currentPage: data.pageNumber,
pageSize: 500,
pageSize: PAGE_SIZE,
paramNameValueMap: {}
});
}, { encodeJSON: true });
console.log({ response });
assureResponse(response);
assert(response.statusCode === 200);
assert(response.body.productList != null); // Missing from stale queued requests?
assert(response.body.productList.length > 0);
assert(response.body.code === 200);
assert(response.body.result?.dataList != null); // Missing from stale queued requests?
assert(response.body.result.dataList.length > 0);
for (let item of response.body.productList) {
for (let item of response.body.result.dataList) {
storeItem({
// NOTE: item.productId seems like the database ID on the website, but item.productCode is the actual LCSC part number used internally for inventory management, so we use that for identification instead
id: `lcsc:product:${item.productCode}`,
@ -92,6 +95,15 @@ module.exports = function ({ session }) {
run: async function (api) {
let { data } = api;
function clean(url) {
// LCSC sometimes uses null, and sometimes uses empty string
if (url == "") {
return null;
} else {
return url;
}
}
createDatasheet(api, {
priority: 0.4,
source: "lcsc",
@ -99,7 +111,8 @@ module.exports = function ({ session }) {
productID: data.productCode,
name: data.productModel,
description: data.productIntroEn,
url: data.pdfUrl
// For TI datasheets, LCSC references an external URL using pdfLinkUrl instead of a local copy
url: clean(data.pdfUrl) ?? clean(data.pdfLinkUrl)
});
}
},

Loading…
Cancel
Save