"use strict" ;
const assert = require ( "assert" ) ;
const assureResponse = require ( "../../shared/assure-response" ) ;
const createDatasheet = require ( "../../shared/create-datasheet" ) ;
// LCSC
// TODO: Validate response formats with validatem instead
const PAGE _SIZE = 100 ; // Maximum amount of items per page that the API allows you to request
module . exports = function ( { session } ) {
return {
seed : [ {
id : "lcsc:home" ,
tags : [ "lcsc:home" ] ,
data : { }
} ] ,
tags : {
"lcsc:home" : [ "lcsc:findCategories" ] ,
"lcsc:category" : [ "lcsc:scrapeCategory" ] ,
"lcsc:product" : [ "lcsc:normalizeProduct" ] ,
} ,
tasks : {
"lcsc:findCategories" : {
ttl : "30d" ,
version : "1" ,
run : async function ( { storeItem } ) {
let response = await session . get ( "https://wmsc.lcsc.com/wmsc/product/catalogs/search" ) ;
assureResponse ( response ) ;
assert ( response . body . result . length > 0 ) ;
assert ( response . body . code === 200 ) ;
function processCategoryEntries ( categories ) {
for ( let category of categories ) {
let productCount = category . productNum ;
let pageCount = Math . ceil ( productCount / PAGE _SIZE ) ;
// NOTE: We do *not* use the page count indicated by the API, but instead calculate it ourself from the product count. This is because the API-specified page count will cap out at the equivalent of 10k items, even when more pages than that are actually available.
for ( let i = 1 ; i <= pageCount ; i ++ ) {
storeItem ( {
id : ` lcsc:category: ${ category . catalogId } :page- ${ i } ` ,
tags : [ "lcsc:category" ] ,
data : {
... category ,
pageNumber : i
}
} ) ;
}
if ( category . childCatelogs != null ) {
processCategoryEntries ( category . childCatelogs ) ;
}
}
}
processCategoryEntries ( response . body . result ) ;
}
} ,
"lcsc:scrapeCategory" : {
ttl : "30d" ,
taskInterval : "1m" ,
run : async function ( { data , storeItem , deleteItem } ) {
let response = await session . post ( ` https://wmsc.lcsc.com/wmsc/product/search/list ` , {
catalogIdList : [ data . catalogId ] ,
currentPage : data . pageNumber ,
pageSize : PAGE _SIZE ,
paramNameValueMap : { }
} , { encodeJSON : true } ) ;
console . log ( { response } ) ;
assureResponse ( response ) ;
assert ( response . body . code === 200 ) ;
assert ( response . body . result ? . dataList != null ) ; // Missing from stale queued requests?
assert ( response . body . result . dataList . length > 0 ) ;
for ( let item of response . body . result . dataList ) {
storeItem ( {
// NOTE: item.productId seems like the database ID on the website, but item.productCode is the actual LCSC part number used internally for inventory management, so we use that for identification instead
id : ` lcsc:product: ${ item . productCode } ` ,
tags : [ "lcsc:product" ] ,
data : item
} ) ;
}
// We don't keep around page items, because the amount of pages for a category can change, and so this isn't a stable identifier. They'll be recreated on the next category scrape anyway.
deleteItem ( ) ;
}
} ,
"lcsc:normalizeProduct" : {
version : "7" ,
parallelTasks : 50 ,
run : async function ( api ) {
let { data } = api ;
function clean ( url ) {
// LCSC sometimes uses null, and sometimes uses empty string
if ( url == "" ) {
return null ;
} else {
return url ;
}
}
createDatasheet ( api , {
priority : 0.4 ,
source : "lcsc" ,
manufacturer : data . brandNameEn ,
productID : data . productCode ,
name : data . productModel ,
description : data . productIntroEn ,
// For TI datasheets, LCSC references an external URL using pdfLinkUrl instead of a local copy
url : clean ( data . pdfUrl ) ? ? clean ( data . pdfLinkUrl )
} ) ;
}
} ,
}
} ;
} ;