scraper-config/lib/tme/task/scrape-product.js

"use strict";

const assert = require("assert");
const cheerio = require("cheerio");
const syncpipe = require("syncpipe");
const url = require("url");
const defaultValue = require("default-value");

const assureResponse = require("../../shared/assure-response");

module.exports = function tmeScrapeProduct({ session }) {
	return async function ({ data, createAlias, updateData, expireDependents }) {
		let response = await session.get(data.url);
		assureResponse(response);

		let $ = cheerio.load(response.body);

		let allMetaHeaders = syncpipe($("h2.o-semantic-only-header").toArray(), [
			(_) => _.map((header) => {
				let $header = $(header);

				return [
					$header.find(".name").text().trim(),
					$header.find(".value").text().trim()
				];
			}),
			(_) => Object.fromEntries(_)
		]);

		let itemData = {
			productID: $(".pip-product-symbol").eq(0).text().trim(),
			manufacturer: $("[data-gtm-event-action='producer_link']").eq(0).text().trim(),
			model: defaultValue(
				allMetaHeaders["Manufacturer part number:"],
				allMetaHeaders["TME Symbol:"]
			),
			description: $(".c-pip__sub-name").eq(0).text().trim(),
			documents: $("a.c-pip__downloads-file-link").toArray()
				.map((link) => {
					let relativeLink = $(link).attr("href");
					
					if (relativeLink != null) {
						return {
							description: $(link).text().trim(),
							url: url.resolve(
								data.url,
								relativeLink
							)
						};
					} else {
						// Probably a video popup
						return null;
					}
				})
				.filter((item) => {
					return item != null;
				}),
			// TODO: Scrape prices
		};

		assert(itemData.productID != null);
		assert(itemData.manufacturer != null);
		assert(itemData.model != null);
		
		createAlias({ from: `tme:product:${itemData.productID}` });

		updateData((oldData) => ({
			... oldData,
			itemData: itemData
		}));

		expireDependents();
	};
};