scraper-config/lib/focus-lcds/task/scrape-category.js

"use strict";

const assureResponse = require("../../shared/assure-response");
const surgeon = require("../../shared/surgeon-utils");

module.exports = function scrapeCategory({ session }) {
	return async function({ data, createItem }) {
		let response = await session.get(data.url);
		assureResponse(response);
		
		let body = response.body.toString();
	
		let nextPageURL = surgeon(`selectMaybeN ".pagination-item--next a" 0 | readAttr href`, body);

		if (nextPageURL != null) {
			createItem({
				id: `focus-lcds:category:${nextPageURL}`,
				tags: [ "focus-lcds:category" ],
				data: { url: nextPageURL }
			});
		}

		let items = surgeon(`selectMany "ul.productList .listItem-title a" | readAttr href`, body);

		for (let url of items) {
			createItem({
				id: `focus-lcds:product:${url}`,
				tags: [ "focus-lcds:product" ],
				data: { url: url }
			});
		}
	};
};