You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
58 lines
1.3 KiB
JavaScript
58 lines
1.3 KiB
JavaScript
'use strict';
|
|
|
|
const createEventEmitter = require("create-event-emitter");
|
|
const defaultValue = require("default-value");
|
|
|
|
function ifNotEmpty(value) {
|
|
if (value !== "") {
|
|
return value;
|
|
}
|
|
}
|
|
|
|
module.exports = function createScraper(options = {}) {
|
|
let scraperSettings = defaultValue(options.scraperSettings, {});
|
|
|
|
const pasteStore = require("./paste-store")();
|
|
const pastebinComScraper = require("./scrapers/pastebin-com")(scraperSettings.pastebinCom);
|
|
const errors = require("./errors");
|
|
|
|
let scraper = createEventEmitter({
|
|
pasteStore: pasteStore
|
|
});
|
|
|
|
pastebinComScraper.on("paste", (paste) => {
|
|
let postedDate = parseInt(paste.date);
|
|
let expiryDate = parseInt(paste.expire);
|
|
|
|
pasteStore.add({
|
|
site: "pastebinCom",
|
|
url: paste.full_url,
|
|
date: postedDate,
|
|
id: paste.key,
|
|
expire: (expiryDate !== 0) ? (expiryDate - postedDate) : undefined,
|
|
username: ifNotEmpty(paste.user),
|
|
language: paste.syntax,
|
|
title: ifNotEmpty(paste.title),
|
|
contents: paste.raw
|
|
});
|
|
});
|
|
|
|
pastebinComScraper.on("error", (error) => {
|
|
if (error instanceof errors.HttpError && error.rateLimited !== true) {
|
|
scraper.emit("warning", error);
|
|
} else {
|
|
scraper.emit("error", error);
|
|
}
|
|
});
|
|
|
|
if (options.scrapePastebinCom) {
|
|
pastebinComScraper.start();
|
|
}
|
|
|
|
pasteStore.on("added", (paste) => {
|
|
scraper.emit("newPaste", paste);
|
|
});
|
|
|
|
return scraper;
|
|
};
|