You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
56 lines
1.2 KiB
JavaScript
56 lines
1.2 KiB
JavaScript
"use strict";
|
|
|
|
const bhttp = require("bhttp");
|
|
|
|
const stScrapeCategory = require("./lib/st/task/scrape-category");
|
|
const stFindCategories = require("./lib/st/task/find-categories");
|
|
const stScrapeProduct = require("./lib/st/task/scrape-product");
|
|
const stNormalizeProduct = require("./lib/st/task/normalize-product");
|
|
|
|
let session = bhttp.session({
|
|
headers: {
|
|
"user-agent": "seekseek.org beta crawler (contact/problems: admin@cryto.net)"
|
|
}
|
|
});
|
|
|
|
let state = { session };
|
|
|
|
module.exports = {
|
|
database: {
|
|
host: "/run/postgresql",
|
|
database: "seekseek_documentation"
|
|
},
|
|
seed: [{
|
|
id: "st:home",
|
|
tags: [ "st:home" ],
|
|
data: {}
|
|
}],
|
|
tags: {
|
|
"st:home": [ "st:findCategories" ],
|
|
"st:category": [ "st:scrapeCategory" ],
|
|
"st:product": [ "st:scrapeProduct", "st:normalizeProduct" ]
|
|
},
|
|
tasks: {
|
|
"st:findCategories": {
|
|
ttl: "15d",
|
|
run: stFindCategories(state)
|
|
},
|
|
"st:scrapeCategory": {
|
|
ttl: "1d",
|
|
taskInterval: "60s",
|
|
version: "2",
|
|
run: stScrapeCategory(state)
|
|
},
|
|
"st:scrapeProduct": {
|
|
ttl: "15d",
|
|
taskInterval: "5s",
|
|
run: stScrapeProduct(state)
|
|
},
|
|
"st:normalizeProduct": {
|
|
dependsOn: [ "st:scrapeProduct" ],
|
|
version: "3",
|
|
run: stNormalizeProduct(state)
|
|
}
|
|
}
|
|
};
|