"use strict" ;
const bhttp = require ( "bhttp" ) ;
const got = require ( "got" ) ;
const assureResponse = require ( "./lib/shared/assure-response" ) ;
let state = {
session : bhttp . session ( {
headers : {
"user-agent" : ( process . env . NODE _ENV === "production" )
? "seekseek.org crawler (seekseek.org/contact)"
: "seekseek.org crawler, development mode (seekseek.org/contact)"
}
} ) ,
// For HTTP/2, until bhttp gains HTTP/2 support
gotSession : got . extend ( {
http2 : true ,
headers : {
// "user-agent": "seekseek.org beta crawler (contact/problems: admin@cryto.net)"
"user-agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0"
} ,
hooks : {
afterResponse : [ ( response ) => {
assureResponse ( response ) ;
return response ;
} ]
}
} )
} ;
module . exports = {
backend : "postgresql" ,
database : {
host : "/run/postgresql" ,
database : "seekseek_documentation" ,
pool : {
max : 75
}
} ,
seed : [ {
id : "st:home" ,
tags : [ "st:home" ] ,
data : { }
} , {
id : "lcsc:home" ,
tags : [ "lcsc:home" ] ,
data : { }
} , {
id : "mouser:sitemap:index" ,
tags : [ "mouser:sitemap" ] ,
data : { url : "https://www.mouser.com/indexgzipwww.xml" }
} , {
id : "tme:sitemap:index" ,
tags : [ "tme:sitemap" ] ,
data : { url : "https://www.tme.eu/en/sitemap.xml" }
// TODO: Delete derived sitemap entries
} , {
id : "farnell:sitemap:index" ,
tags : [ "farnell:sitemap" ] ,
data : { url : "https://uk.farnell.com/sitemap.xml" }
// TODO: Delete derived sitemap entries
} , {
id : "focus-lcds:home" ,
tags : [ "focus-lcds:home" ] ,
data : { }
} ] ,
tags : {
"st:home" : [ "st:findCategories" ] ,
"st:category" : [ "st:scrapeCategory" ] ,
"st:product" : [ "st:scrapeProduct" , "st:normalizeProduct" ] ,
"lcsc:home" : [ "lcsc:findCategories" ] ,
"lcsc:category" : [ "lcsc:scrapeCategory" ] ,
"lcsc:product" : [ "lcsc:normalizeProduct" ] ,
"tme:sitemap" : [ "tme:scrapeSitemap" ] ,
"tme:product" : [ "tme:scrapeProduct" , "tme:normalizeProduct" ] ,
// "farnell:sitemap": [ "farnell:scrapeSitemap" ],
// "farnell:product": [ "farnell:scrapeProduct", "farnell:normalizeProduct" ],
"focus-lcds:home" : [ "focus-lcds:findCategories" ] ,
"focus-lcds:category" : [ "focus-lcds:scrapeCategory" ] ,
"focus-lcds:product" : [ "focus-lcds:scrapeProduct" , "focus-lcds:normalizeProduct" ] ,
} ,
tasks : {
// ST Microelectronics
"st:findCategories" : {
ttl : "15d" ,
run : require ( "./lib/st/task/find-categories" ) ( state )
} ,
"st:scrapeCategory" : {
ttl : "1d" ,
taskInterval : "60s" ,
version : "2" ,
run : require ( "./lib/st/task/scrape-category" ) ( state )
} ,
"st:scrapeProduct" : {
ttl : "15d" ,
taskInterval : "5s" ,
run : require ( "./lib/st/task/scrape-product" ) ( state )
} ,
"st:normalizeProduct" : {
dependsOn : [ "st:scrapeProduct" ] ,
version : "8" ,
parallelTasks : 50 ,
run : require ( "./lib/st/task/normalize-product" ) ( state )
} ,
// LCSC
// FIXME: Commenting out a bunch of tasks but not removing them from the tag assignments will result in an error, but will *not* exit the program. That's probably not right?
"lcsc:findCategories" : {
ttl : "30d" ,
version : "1" ,
run : require ( "./lib/lcsc/task/find-categories" ) ( state )
} ,
"lcsc:scrapeCategory" : {
ttl : "30d" ,
taskInterval : "1m" ,
run : require ( "./lib/lcsc/task/scrape-category" ) ( state )
} ,
"lcsc:normalizeProduct" : {
version : "7" ,
parallelTasks : 50 ,
run : require ( "./lib/lcsc/task/normalize-product" ) ( state )
} ,
// Mouser
"mouser:scrapeSitemap" : {
taskInterval : "30s" ,
run : require ( "./lib/mouser/task/scrape-sitemap" ) ( state )
} ,
// TME.eu
"tme:scrapeSitemap" : {
ttl : "3d" ,
taskInterval : "30s" ,
run : require ( "./lib/tme/task/scrape-sitemap" ) ( state )
} ,
"tme:scrapeProduct" : {
ttl : "60d" ,
taskInterval : "500ms" ,
run : require ( "./lib/tme/task/scrape-product" ) ( state )
} ,
"tme:normalizeProduct" : {
dependsOn : [ "tme:scrapeProduct" ] ,
version : "5" ,
parallelTasks : 50 ,
run : require ( "./lib/tme/task/normalize-product" ) ( state )
} ,
// Farnell
// "farnell:scrapeSitemap": {
// ttl: "3d",
// taskInterval: "30s",
// run: require("./lib/farnell/task/scrape-sitemap")(state)
// },
// "farnell:scrapeProduct": {
// ttl: "60d",
// taskInterval: "500ms",
// run: require("./lib/farnell/task/scrape-product")(state)
// },
// "farnell:normalizeProduct": {
// dependsOn: [ "farnell:scrapeProduct" ],
// version: "1",
// parallelTasks: 50,
// run: require("./lib/farnell/task/normalize-product")(state)
// },
// Focus LCDs
"focus-lcds:findCategories" : {
ttl : "60d" ,
run : require ( "./lib/focus-lcds/task/find-categories" ) ( state )
} ,
"focus-lcds:scrapeCategory" : {
ttl : "15d" ,
taskInterval : "1m" ,
run : require ( "./lib/focus-lcds/task/scrape-category" ) ( state )
} ,
"focus-lcds:scrapeProduct" : {
ttl : "15d" ,
taskInterval : "5s" ,
run : require ( "./lib/focus-lcds/task/scrape-product" ) ( state )
} ,
"focus-lcds:normalizeProduct" : {
dependsOn : [ "focus-lcds:scrapeProduct" ] ,
parallelTasks : 50 ,
run : require ( "./lib/focus-lcds/task/normalize-product" ) ( state )
} ,
}
} ;