You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
114 lines
2.8 KiB
JavaScript
114 lines
2.8 KiB
JavaScript
"use strict";
|
|
|
|
const bhttp = require("bhttp");
|
|
|
|
const stScrapeCategory = require("./lib/st/task/scrape-category");
|
|
const stFindCategories = require("./lib/st/task/find-categories");
|
|
const stScrapeProduct = require("./lib/st/task/scrape-product");
|
|
const stNormalizeProduct = require("./lib/st/task/normalize-product");
|
|
const lcscFindCategories = require("./lib/lcsc/task/find-categories");
|
|
const lcscScrapeCategory = require("./lib/lcsc/task/scrape-category");
|
|
const lcscNormalizeProduct = require("./lib/lcsc/task/normalize-product");
|
|
|
|
let state = {
|
|
session: bhttp.session({
|
|
headers: {
|
|
"user-agent": "seekseek.org beta crawler (contact/problems: admin@cryto.net)"
|
|
}
|
|
})
|
|
};
|
|
|
|
module.exports = {
|
|
database: {
|
|
host: "/run/postgresql",
|
|
database: "seekseek_documentation"
|
|
},
|
|
seed: [{
|
|
id: "st:home",
|
|
tags: [ "st:home" ],
|
|
data: {}
|
|
}, {
|
|
id: "lcsc:home",
|
|
tags: [ "lcsc:home" ],
|
|
data: {}
|
|
}, {
|
|
id: "mouser:sitemap:index",
|
|
tags: [ "mouser:sitemap" ],
|
|
data: { url: "https://www.mouser.com/indexgzipwww.xml" }
|
|
}, {
|
|
id: "tme:sitemap:index",
|
|
tags: [ "tme:sitemap" ],
|
|
data: { url: "https://www.tme.eu/en/sitemap.xml" }
|
|
}],
|
|
tags: {
|
|
"st:home": [ "st:findCategories" ],
|
|
"st:category": [ "st:scrapeCategory" ],
|
|
"st:product": [ "st:scrapeProduct", "st:normalizeProduct" ],
|
|
"lcsc:home": [ "lcsc:findCategories" ],
|
|
"lcsc:category": [ "lcsc:scrapeCategory" ],
|
|
"lcsc:product": [ "lcsc:normalizeProduct" ],
|
|
"tme:sitemap": [ "tme:scrapeSitemap" ],
|
|
"tme:product": [ "tme:scrapeProduct", "tme:normalizeProduct" ],
|
|
},
|
|
tasks: {
|
|
"st:findCategories": {
|
|
ttl: "15d",
|
|
run: stFindCategories(state)
|
|
},
|
|
"st:scrapeCategory": {
|
|
ttl: "1d",
|
|
taskInterval: "60s",
|
|
version: "2",
|
|
run: stScrapeCategory(state)
|
|
},
|
|
"st:scrapeProduct": {
|
|
ttl: "15d",
|
|
taskInterval: "5s",
|
|
run: stScrapeProduct(state)
|
|
},
|
|
"st:normalizeProduct": {
|
|
dependsOn: [ "st:scrapeProduct" ],
|
|
version: "8",
|
|
parallelTasks: Infinity,
|
|
run: stNormalizeProduct(state)
|
|
},
|
|
"lcsc:findCategories": {
|
|
ttl: "30d",
|
|
taskVersion: "1",
|
|
run: lcscFindCategories(state)
|
|
},
|
|
"lcsc:scrapeCategory": {
|
|
ttl: "30d",
|
|
taskInterval: "1m",
|
|
run: lcscScrapeCategory(state)
|
|
},
|
|
"lcsc:normalizeProduct": {
|
|
version: "7",
|
|
parallelTasks: Infinity,
|
|
run: lcscNormalizeProduct(state)
|
|
},
|
|
"mouser:scrapeSitemap": {
|
|
taskInterval: "30s",
|
|
run: require("./lib/mouser/task/scrape-sitemap")(state)
|
|
},
|
|
"tme:scrapeSitemap": {
|
|
ttl: "3d",
|
|
taskInterval: "30s",
|
|
taskVersion: "1",
|
|
run: require("./lib/tme/task/scrape-sitemap")(state)
|
|
},
|
|
"tme:scrapeProduct": {
|
|
ttl: "60d",
|
|
taskInterval: "1s",
|
|
taskVersion: "1",
|
|
run: require("./lib/tme/task/scrape-product")(state)
|
|
},
|
|
"tme:normalizeProduct": {
|
|
dependsOn: [ "tme:scrapeProduct" ],
|
|
version: "3",
|
|
parallelTasks: Infinity,
|
|
run: require("./lib/tme/task/normalize-product")(state)
|
|
},
|
|
}
|
|
};
|