diff --git a/src/sync/update-stream.js b/src/sync/update-stream.js index a159e78..f182858 100644 --- a/src/sync/update-stream.js +++ b/src/sync/update-stream.js @@ -14,6 +14,7 @@ const createNDJSONParseStream = require("./ndjson-parse-stream"); module.exports = function({ scrapingHost }) { return function createUpdateStream({ since, prefix } = {}) { let lastTimestamp = since ?? new Date(0); + let resultCounter = 0; return pipe([ simpleSource(() => { @@ -27,6 +28,7 @@ module.exports = function({ scrapingHost }) { }); }).then((response) => { if (response.statusCode === 200) { + console.log("Received success response from sync server"); return fromNodeStream.fromReadable(response); } else { throw new Error(`Got unexpected status code ${response.statusCode}`); @@ -48,6 +50,12 @@ module.exports = function({ scrapingHost }) { createCombineSequentialStream(), createNDJSONParseStream(), createSpyStream((item) => { + resultCounter++; + + if (resultCounter % 1000 === 0) { + console.log(`Received ${resultCounter} items since last restart`); + } + if (item.updatedAt != null) { // TODO: Can this be made significantly more performant by string-sorting the timestamps in ISO format directly, instead of going through a parsing cycle? let itemDate = new Date(item.updatedAt);