Log incoming synced items

master
Sven Slootweg 1 year ago
parent c21a9fea09
commit 69811718ba

@ -14,6 +14,7 @@ const createNDJSONParseStream = require("./ndjson-parse-stream");
module.exports = function({ scrapingHost }) {
return function createUpdateStream({ since, prefix } = {}) {
let lastTimestamp = since ?? new Date(0);
let resultCounter = 0;
return pipe([
simpleSource(() => {
@ -27,6 +28,7 @@ module.exports = function({ scrapingHost }) {
});
}).then((response) => {
if (response.statusCode === 200) {
console.log("Received success response from sync server");
return fromNodeStream.fromReadable(response);
} else {
throw new Error(`Got unexpected status code ${response.statusCode}`);
@ -48,6 +50,12 @@ module.exports = function({ scrapingHost }) {
createCombineSequentialStream(),
createNDJSONParseStream(),
createSpyStream((item) => {
resultCounter++;
if (resultCounter % 1000 === 0) {
console.log(`Received ${resultCounter} items since last restart`);
}
if (item.updatedAt != null) {
// TODO: Can this be made significantly more performant by string-sorting the timestamps in ISO format directly, instead of going through a parsing cycle?
let itemDate = new Date(item.updatedAt);

Loading…
Cancel
Save