From 69811718bafa6c8c5317804cc152809beef01510 Mon Sep 17 00:00:00 2001
From: Sven Slootweg <admin@cryto.net>
Date: Sun, 27 Nov 2022 00:41:33 +0100
Subject: [PATCH] Log incoming synced items

---
 src/sync/update-stream.js | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/sync/update-stream.js b/src/sync/update-stream.js
index a159e78..f182858 100644
--- a/src/sync/update-stream.js
+++ b/src/sync/update-stream.js
@@ -14,6 +14,7 @@ const createNDJSONParseStream = require("./ndjson-parse-stream");
 module.exports = function({ scrapingHost }) {
 	return function createUpdateStream({ since, prefix } = {}) {
 		let lastTimestamp = since ?? new Date(0);
+		let resultCounter = 0;
 		
 		return pipe([
 			simpleSource(() => {
@@ -27,6 +28,7 @@ module.exports = function({ scrapingHost }) {
 						});
 					}).then((response) => {
 						if (response.statusCode === 200) {
+							console.log("Received success response from sync server");
 							return fromNodeStream.fromReadable(response);
 						} else {
 							throw new Error(`Got unexpected status code ${response.statusCode}`);
@@ -48,6 +50,12 @@ module.exports = function({ scrapingHost }) {
 			createCombineSequentialStream(),
 			createNDJSONParseStream(),
 			createSpyStream((item) => {
+				resultCounter++;
+
+				if (resultCounter % 1000 === 0) {
+					console.log(`Received ${resultCounter} items since last restart`);
+				}
+				
 				if (item.updatedAt != null) {
 					// TODO: Can this be made significantly more performant by string-sorting the timestamps in ISO format directly, instead of going through a parsing cycle?
 					let itemDate = new Date(item.updatedAt);