"use strict"; const assert = require("assert"); const syncpipe = require("syncpipe"); const assureResponse = require("../../shared/assure-response"); const extractId = require("../extract-id"); module.exports = function scrapeCategory({ session }) { return async function({ data, createItem }) { let response = await session.get(data.url); assureResponse(response); let prmisID = extractId(response.body.toString()); let listingResponse = await session.get(`https://www.st.com/en/documentation/scraper.cxst-ps-grid.html/${encodeURIComponent(prmisID)}.json`, { noDecode: true }); assureResponse(response); let listingBuffer = listingResponse.body; if (listingBuffer.length > 0) { // This is a category that has a product explorer let listing = JSON.parse(listingBuffer.toString()); let cellNames = listing.columns.map((column) => { let cellName = (column.identifier != null) ? `${column.identifier}_${column.qualifier_identifier}` : `nonstandard:${column.name}:${column.qualifier}` createItem({ id: `st:column:${cellName}`, tags: [ "st:column" ], data: column }); return cellName; }); for (let row of listing.rows) { assert(row.productId != null); let cellData = syncpipe(row.cells, [ (_) => _.map((cell, i) => [ cellNames[i], cell.value ]), (_) => Object.fromEntries(_) ]); createItem({ id: `st:product:${row.productId}`, tags: [ "st:product" ], data: { ... row, cells: undefined, cellData: cellData } }); } } else { console.warn("Warning: empty response, category does not have product explorer"); } }; };