"use strict"; const assert = require("assert"); const assureResponse = require("../../shared/assure-response"); const createDatasheet = require("../../shared/create-datasheet"); // LCSC // TODO: Validate response formats with validatem instead module.exports = function ({ session }) { return { seed: [{ id: "lcsc:home", tags: [ "lcsc:home" ], data: {} }], tags: { "lcsc:home": [ "lcsc:findCategories" ], "lcsc:category": [ "lcsc:scrapeCategory" ], "lcsc:product": [ "lcsc:normalizeProduct" ], }, tasks: { "lcsc:findCategories": { ttl: "30d", version: "1", run: async function ({ storeItem }) { let response = await session.get("https://wwwapi.lcsc.com/v1/home/category"); assureResponse(response); assert(response.body.length > 0); assert(response.statusCode === 200); function processCategoryEntries(categories) { for (let category of categories) { let productCount = category.productNum; let pageCount = Math.ceil(productCount / 500); // NOTE: We do *not* use the page count indicated by the API, but instead calculate it ourself from the product count. This is because the API-specified page count will cap out at the equivalent of 10k items, even when more pages than that are actually available. for (let i = 1; i <= pageCount; i++) { storeItem({ id: `lcsc:category:${category.catalogId}:page-${i}`, tags: [ "lcsc:category" ], data: { ... category, pageNumber: i } }); } if (category.childCatelogs != null) { processCategoryEntries(category.childCatelogs); } } } processCategoryEntries(response.body); } }, "lcsc:scrapeCategory": { ttl: "30d", taskInterval: "1m", run: async function ({ data, storeItem, deleteItem }) { let response = await session.post(`https://wwwapi.lcsc.com/v1/products/list`, { catalogIdList: [ data.catalogId ], currentPage: data.pageNumber, pageSize: 500, paramNameValueMap: {} }); assureResponse(response); assert(response.statusCode === 200); assert(response.body.productList != null); // Missing from stale queued requests? assert(response.body.productList.length > 0); for (let item of response.body.productList) { storeItem({ // NOTE: item.productId seems like the database ID on the website, but item.productCode is the actual LCSC part number used internally for inventory management, so we use that for identification instead id: `lcsc:product:${item.productCode}`, tags: [ "lcsc:product" ], data: item }); } // We don't keep around page items, because the amount of pages for a category can change, and so this isn't a stable identifier. They'll be recreated on the next category scrape anyway. deleteItem(); } }, "lcsc:normalizeProduct": { version: "7", parallelTasks: 50, run: async function (api) { let { data } = api; createDatasheet(api, { priority: 0.4, source: "lcsc", manufacturer: data.brandNameEn, productID: data.productCode, name: data.productModel, description: data.productIntroEn, url: data.pdfUrl }); } }, } }; };