"use strict"; const assert = require("assert"); const assureResponse = require("../../shared/assure-response"); const createDatasheet = require("../../shared/create-datasheet"); // LCSC // TODO: Validate response formats with validatem instead const PAGE_SIZE = 100; // Maximum amount of items per page that the API allows you to request module.exports = function ({ session }) { return { seed: [{ id: "lcsc:home", tags: [ "lcsc:home" ], data: {} }], tags: { "lcsc:home": [ "lcsc:findCategories" ], "lcsc:category": [ "lcsc:scrapeCategory" ], "lcsc:product": [ "lcsc:normalizeProduct" ], }, tasks: { "lcsc:findCategories": { ttl: "30d", version: "1", run: async function ({ storeItem }) { let response = await session.get("https://wmsc.lcsc.com/wmsc/product/catalogs/search"); assureResponse(response); assert(response.body.result.length > 0); assert(response.body.code === 200); function processCategoryEntries(categories) { for (let category of categories) { let productCount = category.productNum; let pageCount = Math.ceil(productCount / PAGE_SIZE); // NOTE: We do *not* use the page count indicated by the API, but instead calculate it ourself from the product count. This is because the API-specified page count will cap out at the equivalent of 10k items, even when more pages than that are actually available. for (let i = 1; i <= pageCount; i++) { storeItem({ id: `lcsc:category:${category.catalogId}:page-${i}`, tags: [ "lcsc:category" ], data: { ... category, pageNumber: i } }); } if (category.childCatelogs != null) { processCategoryEntries(category.childCatelogs); } } } processCategoryEntries(response.body.result); } }, "lcsc:scrapeCategory": { ttl: "30d", taskInterval: "1m", run: async function ({ data, storeItem, deleteItem }) { let response = await session.post(`https://wmsc.lcsc.com/wmsc/product/search/list`, { catalogIdList: [ data.catalogId ], currentPage: data.pageNumber, pageSize: PAGE_SIZE, paramNameValueMap: {} }, { encodeJSON: true }); console.log({ response }); assureResponse(response); assert(response.body.code === 200); assert(response.body.result?.dataList != null); // Missing from stale queued requests? assert(response.body.result.dataList.length > 0); for (let item of response.body.result.dataList) { storeItem({ // NOTE: item.productId seems like the database ID on the website, but item.productCode is the actual LCSC part number used internally for inventory management, so we use that for identification instead id: `lcsc:product:${item.productCode}`, tags: [ "lcsc:product" ], data: item }); } // We don't keep around page items, because the amount of pages for a category can change, and so this isn't a stable identifier. They'll be recreated on the next category scrape anyway. deleteItem(); } }, "lcsc:normalizeProduct": { version: "7", parallelTasks: 50, run: async function (api) { let { data } = api; function clean(url) { // LCSC sometimes uses null, and sometimes uses empty string if (url == "") { return null; } else { return url; } } createDatasheet(api, { priority: 0.4, source: "lcsc", manufacturer: data.brandNameEn, productID: data.productCode, name: data.productModel, description: data.productIntroEn, // For TI datasheets, LCSC references an external URL using pdfLinkUrl instead of a local copy url: clean(data.pdfUrl) ?? clean(data.pdfLinkUrl) }); } }, } }; };