You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
107 lines
2.8 KiB
JavaScript
107 lines
2.8 KiB
JavaScript
"use strict";
|
|
|
|
const debug = require("debug")("srap:backend:postgresql:query:get-task-stream");
|
|
const simpleSource = require("@promistream/simple-source");
|
|
|
|
const query = `
|
|
WITH
|
|
dependency_tasks AS (
|
|
SELECT * FROM
|
|
json_to_recordset(:dependencyTaskDefinitions) AS x(task text, task_version text)
|
|
),
|
|
matching_items AS (
|
|
SELECT
|
|
DISTINCT ON (srap_items.id)
|
|
srap_items.*,
|
|
results.updated_at AS result_date,
|
|
results.task_version,
|
|
(
|
|
results.is_successful = TRUE
|
|
AND (
|
|
results.expires_at < NOW()
|
|
OR results.is_invalidated = TRUE
|
|
)
|
|
) AS is_candidate
|
|
FROM srap_items
|
|
INNER JOIN srap_tags
|
|
ON srap_tags.item_id = srap_items.id
|
|
AND srap_tags.name = ANY(:tags)
|
|
LEFT JOIN srap_task_results AS results
|
|
ON results.item_id = srap_items.id
|
|
AND results.task = :task
|
|
WHERE
|
|
NOT EXISTS (
|
|
SELECT FROM srap_tasks_in_progress AS pr WHERE pr.item_id = srap_items.id
|
|
)
|
|
),
|
|
candidates AS (
|
|
SELECT * FROM matching_items
|
|
WHERE result_date IS NULL
|
|
UNION
|
|
SELECT * FROM matching_items
|
|
WHERE is_candidate = TRUE
|
|
OR NOT (task_version = :taskVersion)
|
|
)
|
|
(
|
|
SELECT
|
|
*
|
|
FROM
|
|
candidates
|
|
WHERE
|
|
NOT EXISTS (
|
|
SELECT
|
|
results.*
|
|
FROM dependency_tasks
|
|
LEFT JOIN srap_task_results AS results
|
|
ON dependency_tasks.task = results.task
|
|
AND dependency_tasks.task_version = results.task_version
|
|
AND results.item_id = candidates.id
|
|
WHERE
|
|
results.is_successful IS NULL
|
|
OR results.is_successful = FALSE
|
|
OR (
|
|
results.is_successful = TRUE
|
|
AND (
|
|
results.expires_at < NOW()
|
|
OR results.is_invalidated = TRUE
|
|
)
|
|
)
|
|
)
|
|
) LIMIT :resultLimit;
|
|
`;
|
|
|
|
module.exports = function ({ metrics, backendSettings }) {
|
|
return function (tx, { task }) {
|
|
return simpleSource(() => {
|
|
let startTime = Date.now();
|
|
|
|
return Promise.try(() => {
|
|
return tx.raw(query, {
|
|
tags: task.tags,
|
|
task: task.name,
|
|
taskVersion: task.version,
|
|
resultLimit: backendSettings.taskBatchSize,
|
|
dependencyTaskDefinitions: JSON.stringify(task.dependencies.map((dependency) => {
|
|
// Case-mapping for SQL compatibility
|
|
return { task_version: dependency.version, task: dependency.name };
|
|
}))
|
|
});
|
|
}).then((result) => {
|
|
let timeElapsed = Date.now() - startTime;
|
|
|
|
metrics.taskFetchTime.labels({ task: task.name }).set(timeElapsed / 1000);
|
|
metrics.taskFetchResults.labels({ task: task.name }).set(result.rowCount);
|
|
|
|
debug(`Task retrieval query for '${task.name}' took ${timeElapsed}ms and produced ${result.rowCount} results`);
|
|
|
|
if (result.rowCount > 0) {
|
|
return result.rows;
|
|
} else {
|
|
// TODO: Consider using LISTEN/NOTIFY instead?
|
|
return Promise.resolve([]).delay(backendSettings.taskBatchDelay);
|
|
}
|
|
});
|
|
});
|
|
};
|
|
}
|