You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
61 lines
1.6 KiB
JavaScript
61 lines
1.6 KiB
JavaScript
"use strict";
|
|
|
|
const assert = require("assert");
|
|
const syncpipe = require("syncpipe");
|
|
|
|
const assureResponse = require("../../shared/assure-response");
|
|
const extractId = require("../extract-id");
|
|
|
|
module.exports = function scrapeCategory({ session }) {
|
|
return async function({ data, createItem }) {
|
|
let response = await session.get(data.url);
|
|
assureResponse(response);
|
|
|
|
let prmisID = extractId(response.body.toString());
|
|
|
|
let listingResponse = await session.get(`https://www.st.com/en/documentation/scraper.cxst-ps-grid.html/${encodeURIComponent(prmisID)}.json`, { noDecode: true });
|
|
assureResponse(response);
|
|
let listingBuffer = listingResponse.body;
|
|
|
|
if (listingBuffer.length > 0) {
|
|
// This is a category that has a product explorer
|
|
let listing = JSON.parse(listingBuffer.toString());
|
|
|
|
let cellNames = listing.columns.map((column) => {
|
|
let cellName = (column.identifier != null)
|
|
? `${column.identifier}_${column.qualifier_identifier}`
|
|
: `nonstandard:${column.name}:${column.qualifier}`
|
|
|
|
createItem({
|
|
id: `st:column:${cellName}`,
|
|
tags: [ "st:column" ],
|
|
data: column
|
|
});
|
|
|
|
return cellName;
|
|
});
|
|
|
|
for (let row of listing.rows) {
|
|
assert(row.productId != null);
|
|
|
|
let cellData = syncpipe(row.cells, [
|
|
(_) => _.map((cell, i) => [ cellNames[i], cell.value ]),
|
|
(_) => Object.fromEntries(_)
|
|
]);
|
|
|
|
createItem({
|
|
id: `st:product:${row.productId}`,
|
|
tags: [ "st:product" ],
|
|
data: {
|
|
... row,
|
|
cells: undefined,
|
|
cellData: cellData
|
|
}
|
|
});
|
|
}
|
|
} else {
|
|
console.warn("Warning: empty response, category does not have product explorer");
|
|
}
|
|
};
|
|
};
|