Make scraping host configurable

src/app.js

@@ -6,9 +6,15 @@ const path = require("path");
 const defaultValue = require("default-value");
 const moize = require("moize");
 
+const config = require("../config.json");
 const knex = require("knex")(require("../knexfile"));
 
-const createSynchronizer = require("./sync")({ knex: knex });
+let state = {
+	knex: knex,
+	scrapingHost: config.scrapingHost
+};
+
+const createSynchronizer = require("./sync")(state);
 
 createSynchronizer("datasheets_products", "datasheet:", (item) => {
 	if (item.data.url != null) {

src/sync/index.js

@@ -4,9 +4,12 @@ const Promise = require("bluebird");
 const matchValue = require("match-value");
 const pipe = require("@promistream/pipe");
 const simpleSink = require("@promistream/simple-sink");
-const updateStream = require("./update-stream");
 
-module.exports = function ({ knex }) {
+module.exports = function (state) {
+	const updateStream = require("./update-stream")(state);
+
+	let { knex } = state;
+
 	return function createSynchronizer(tableName, prefix, mapper, { getLastTimestamp } = {}) {
 		return Promise.try(() => {
 			if (getLastTimestamp != null) {

src/sync/update-stream.js

@@ -12,7 +12,8 @@ const fromNodeStream = require("@promistream/from-node-stream");
 
 const createNDJSONParseStream = require("./ndjson-parse-stream");
 
-module.exports = function createUpdateStream({ since, prefix } = {}) {
+module.exports = function({ scrapingHost }) {
+	return function createUpdateStream({ since, prefix } = {}) {
 		let lastTimestamp = since ?? new Date(0);
 		
 		return pipe([
@@ -22,7 +23,7 @@ module.exports = function createUpdateStream({ since, prefix } = {}) {
 						// To ensure that we don't hammer the srap instance
 						return Promise.delay(5 * 1000);
 					}).then(() => {
-					return bhttp.get(`http://localhost:3000/updates?prefix=${encodeURIComponent(prefix)}&since=${Math.floor(lastTimestamp.getTime())}`, {
+						return bhttp.get(`http://${scrapingHost}/updates?prefix=${encodeURIComponent(prefix)}&since=${Math.floor(lastTimestamp.getTime())}`, {
 							stream: true
 						});
 					}).then((response) => {
@@ -58,4 +59,5 @@ module.exports = function createUpdateStream({ since, prefix } = {}) {
 				}
 			})
 		]);
+	};
 };