You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
74 lines
2.2 KiB
JavaScript
74 lines
2.2 KiB
JavaScript
"use strict";
|
|
|
|
const assert = require("assert");
|
|
|
|
const assureResponse = require("../../shared/assure-response");
|
|
|
|
// TODO: Validate response formats with validatem instead
|
|
|
|
module.exports = function lcscScrapeCategory(state) {
|
|
const withCSRFToken = require("../with-csrf-token")(state);
|
|
|
|
let { session } = state;
|
|
|
|
return async function ({ data, createItem, deleteItem, updateData }) {
|
|
let response = await withCSRFToken((token) => {
|
|
return session.post(`https://lcsc.com/api/products/search`, {
|
|
current_page: String(data.pageNumber),
|
|
category: String(data.id),
|
|
in_stock: "false",
|
|
is_RoHS: "false",
|
|
show_icon: "false",
|
|
search_content: "",
|
|
limit: "10000"
|
|
}, {
|
|
headers: {
|
|
"accept": "application/json, text/javascript, */*; q=0.01",
|
|
"X-CSRF-TOKEN": token
|
|
}
|
|
});
|
|
});
|
|
|
|
assureResponse(response);
|
|
assert(response.body.code === 200);
|
|
assert(response.body.result.data != null);
|
|
|
|
for (let item of response.body.result.data) {
|
|
createItem({
|
|
// NOTE: item.id seems like the database ID on the website, but item.number is the actual LCSC part number used internally for inventory management, so we use that for identification instead
|
|
id: `lcsc:product:${item.number}`,
|
|
tags: [ "lcsc:product" ],
|
|
data: item
|
|
});
|
|
}
|
|
|
|
if (data.pageNumber === 1) {
|
|
let totalPageCount = response.body.result.total_page;
|
|
assert(totalPageCount != null);
|
|
|
|
updateData((data) => ({
|
|
... data,
|
|
pageCount: totalPageCount
|
|
}));
|
|
|
|
// for (let i = 2; i <= totalPageCount; i++) {
|
|
// createItem({
|
|
// id: `lcsc:category:${data.id}:page-${i}`,
|
|
// tags: [ "lcsc:category" ],
|
|
// data: {
|
|
// id: data.id,
|
|
// pageNumber: i
|
|
// }
|
|
// });
|
|
// }
|
|
// FIXME: Figure out a workaround for the 10k-items-per-category cap
|
|
if (totalPageCount > 1) {
|
|
console.warn(`WARNING (LCSC): More than one page for category ${data.id}, but cannot paginate!`);
|
|
}
|
|
} else {
|
|
// We don't keep around items representing pages beyond the first, after indexing them, because total page count can change and the page numbers are not stable identifiers. We can just recreate them on the next scrape of the first page (which always exists).
|
|
deleteItem();
|
|
}
|
|
};
|
|
};
|