"use strict"; const debug = require("debug")("srap:backend:postgresql:query:get-task-stream"); const simpleSource = require("@promistream/simple-source"); const query = ` WITH dependency_tasks AS ( SELECT * FROM json_to_recordset(:dependencyTaskDefinitions) AS x(task text, task_version text) ), matching_items AS ( SELECT DISTINCT ON (srap_items.id) srap_items.*, results.updated_at AS result_date, results.task_version, ( results.is_successful = TRUE AND ( results.expires_at < NOW() OR results.is_invalidated = TRUE ) ) AS is_candidate FROM srap_items INNER JOIN srap_tags ON srap_tags.item_id = srap_items.id AND srap_tags.name = ANY(:tags) LEFT JOIN srap_task_results AS results ON results.item_id = srap_items.id AND results.task = :task WHERE NOT EXISTS ( SELECT FROM srap_tasks_in_progress AS pr WHERE pr.item_id = srap_items.id ) ), candidates AS ( SELECT * FROM matching_items WHERE result_date IS NULL UNION SELECT * FROM matching_items WHERE is_candidate = TRUE OR NOT (task_version = :taskVersion) ) ( SELECT * FROM candidates WHERE NOT EXISTS ( SELECT results.* FROM dependency_tasks LEFT JOIN srap_task_results AS results ON dependency_tasks.task = results.task AND dependency_tasks.task_version = results.task_version AND results.item_id = candidates.id WHERE results.is_successful IS NULL OR results.is_successful = FALSE OR ( results.is_successful = TRUE AND ( results.expires_at < NOW() OR results.is_invalidated = TRUE ) ) ) ) LIMIT :resultLimit; `; module.exports = function ({ metrics, backendSettings }) { return function (tx, { task }) { return simpleSource(() => { let startTime = Date.now(); return Promise.try(() => { return tx.raw(query, { tags: task.tags, task: task.name, taskVersion: task.version, resultLimit: backendSettings.taskBatchSize, dependencyTaskDefinitions: JSON.stringify(task.dependencies.map((dependency) => { // Case-mapping for SQL compatibility return { task_version: dependency.version, task: dependency.name }; })) }); }).then((result) => { let timeElapsed = Date.now() - startTime; metrics.taskFetchTime.labels({ task: task.name }).set(timeElapsed / 1000); metrics.taskFetchResults.labels({ task: task.name }).set(result.rowCount); debug(`Task retrieval query for '${task.name}' took ${timeElapsed}ms and produced ${result.rowCount} results`); if (result.rowCount > 0) { return result.rows; } else { // TODO: Consider using LISTEN/NOTIFY instead? return Promise.resolve([]).delay(backendSettings.taskBatchDelay); } }); }); }; }