Rewrite queue refill query to be much faster, fix duplicate tags

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 junk
+.clinic

migrations/20221125234414_unique-tags.js

@@ -0,0 +1,39 @@
+"use strict";
+
+module.exports.up = function(knex, Promise) {
+	// Get rid of existing duplicate entries
+	return knex.raw(`
+		DELETE FROM srap_tags
+		WHERE id IN (
+			SELECT
+				id
+			FROM (
+				SELECT
+					id,
+					row_number() OVER w as rnum
+				FROM srap_tags
+				WINDOW w AS (
+					PARTITION BY name, item_id
+					ORDER BY id
+				)
+		
+			) t
+		WHERE t.rnum > 1);
+	`).then(() => {
+		return knex.schema
+			.alterTable("srap_tags", (table) => {
+				table.dropPrimary();
+				table.dropIndex("name");
+				table.dropColumn("id");
+				table.primary([ "name", "item_id" ]);
+			});
+	});
+};
+	
+module.exports.down = function(knex, Promise) {
+	return knex.schema
+		.alterTable("srap_tags", (table) => {
+			table.dropPrimary();
+			table.bigIncrements("id").primary();
+		});
+};

package.json

@@ -61,7 +61,8 @@
   },
   "devDependencies": {
     "@joepie91/eslint-config": "^1.1.0",
-    "eslint": "^7.21.0"
+    "eslint": "^7.21.0",
+    "clinic": "^12.0.0"
   },
   "bin": {
     "srap-server": "./bin/server",

src/database-backends/postgresql/index.js

@@ -167,7 +167,7 @@ module.exports = function(state) {
 								id: actualID,
 								data: update(existingData),
 								createdBy: parentID,
-								tags: tags.map((tag) => ({ name: tag })),
+								tags: tags.map((tag) => ({ name: tag, itemId: actualID })),
 								aliases: allAliases.map((alias) => ({ alias: alias })),
 								updatedAt: new Date()
 							};

src/database-backends/postgresql/models/tag.js

@@ -5,6 +5,7 @@ const { Model, QueryBuilder } = require("objection");
 module.exports = function ({ db }) {
 	return class Tag extends Model {
 		static tableName = "srap_tags";
+		static idColumn = [ "name", "itemId" ];
 	
 		static get relationMappings() {
 			return {

src/database-backends/postgresql/queries/get-task-stream.js

@@ -19,79 +19,82 @@ const fetchQuery = `
 	LIMIT :resultLimit
 `;
 
-const fillQuery = `
+function makeFillQuery(withDependencies) {
-	WITH
+	return `
-		dependency_tasks AS (
+		WITH
-			SELECT * FROM
+			${withDependencies ? `
-				json_to_recordset(:dependencyTaskDefinitions) AS x(task text, task_version text)
+				dependencies AS (
-		),
+					SELECT * FROM json_to_recordset(:dependencyTaskDefinitions) AS x(task text, task_version text)
-		matching_items AS (
+				),
-			SELECT
+				satisfied AS (
-				srap_items.*,
+					SELECT results.* FROM dependencies
-				results.updated_at AS result_date,
+					LEFT JOIN srap_task_results AS results
-				results.task_version,
+						ON dependencies.task = results.task
-				(
+						AND dependencies.task_version = results.task_version
-					results.is_successful = TRUE
+					WHERE
-					AND (
+						results.is_successful = TRUE
-						results.expires_at < NOW()
+						AND results.is_invalidated = FALSE
-						OR results.is_invalidated = TRUE
+						AND results.expires_at > NOW()
-					)
+				),
-				) AS is_candidate
+				counts AS (
-			FROM srap_items
+					SELECT item_id, COUNT(task) AS count FROM satisfied GROUP BY item_id
-				INNER JOIN srap_tags
+				),
-					ON srap_tags.item_id = srap_items.id
+				dependency_candidates AS (
-					AND srap_tags.name = ANY(:tags)
+					SELECT item_id FROM counts WHERE count = :dependencyCount
-				LEFT JOIN srap_task_results AS results
+				),
-					ON results.item_id = srap_items.id
+			` : "" }
-					AND results.task = :task
+			tag_candidates AS (
-		),
+				SELECT item_id FROM srap_tags WHERE name = ANY(:tags)
-		candidates AS (
+			),
-			SELECT * FROM matching_items
+			full_candidates AS MATERIALIZED (
-				WHERE result_date IS NULL
+				${withDependencies
-			UNION ALL
+					? `
-			SELECT * FROM matching_items
+						SELECT tag_candidates.item_id FROM dependency_candidates
-				WHERE is_candidate = TRUE
+						INNER JOIN tag_candidates
-				OR NOT (task_version = :taskVersion)
+							ON dependency_candidates.item_id = tag_candidates.item_id
-		)
+					`
-	(
+					: `
+						SELECT item_id FROM tag_candidates
+					`
+				}
+			)
 		SELECT
 			:task AS task,
-			id AS item_id
+			item_id
-		FROM
+		FROM full_candidates
-			candidates
+		WHERE NOT EXISTS (
-		WHERE
+			SELECT item_id FROM srap_task_results AS results
-			NOT EXISTS (
+			WHERE
-				SELECT
+				item_id = full_candidates.item_id
-					results.*
+				AND results.task = :task
-				FROM dependency_tasks
+				AND results.task_version = :taskVersion
-					LEFT JOIN srap_task_results AS results
+				AND results.is_successful = TRUE
-						ON dependency_tasks.task = results.task
+				AND results.is_invalidated = FALSE
-						AND dependency_tasks.task_version = results.task_version
+				AND results.expires_at > NOW()
-						AND results.item_id = candidates.id
+		)
-				WHERE
+	`;
-					results.is_successful IS NULL
+}
-					OR results.is_successful = FALSE
+
-					OR (
+const fillQueryWithDependencies = makeFillQuery(true);
-						results.is_successful = TRUE
+const fillQueryWithoutDependencies = makeFillQuery(false);
-						AND (
-							results.expires_at < NOW()
-							OR results.is_invalidated = TRUE
-						)
-					)
-			)
-	)
-`;
 
 module.exports = function ({ metrics, backendSettings, knex }) {
 	return function (tx, { task }) {
+		let hasDependencies = (task.dependencies.length > 0);
+
 		let refillParameters = {
 			tags: task.tags,
 			task: task.name,
 			taskVersion: task.version,
-			dependencyTaskDefinitions: JSON.stringify(task.dependencies.map((dependency) => {
+			... hasDependencies
-				// Case-mapping for SQL compatibility
+				? {
-				return { task_version: dependency.version, task: dependency.name };
+					dependencyCount: task.dependencies.length,
-			}))
+					dependencyTaskDefinitions: JSON.stringify(task.dependencies.map((dependency) => {
+						// Case-mapping for SQL compatibility
+						return { task_version: dependency.version, task: dependency.name };
+					}))
+				}
+				: {}
 		};
 
 		let fetchParameters = {
@@ -104,6 +107,10 @@ module.exports = function ({ metrics, backendSettings, knex }) {
 			let startTime = Date.now();
 
 			return Promise.try(() => {
+				let fillQuery = (hasDependencies)
+					? fillQueryWithDependencies
+					: fillQueryWithoutDependencies;
+
 				// NOTE: We are deliberately bypassing the transaction here! Also deliberately not using VALUES, since we're inserting from the results of another query instead
 				return knex.raw(`
 					INSERT INTO srap_queue (task, item_id)