backend-refactor
Sven Slootweg 3 years ago
parent 464853bd7c
commit 6b6ab7332e

@ -293,3 +293,22 @@ SELECT failures.* FROM failures
ON failures.task_result_id = task_results.id ON failures.task_result_id = task_results.id
WHERE WHERE
task_results.operation = :operation task_results.operation = :operation
==================
task_states
all task_results, with is_candidate precalculated
dependency_task_states
all task_states for tasks+versions listed in :dependencyTasks
candidates (items which are *permitted* to be run)
all tag-relevant items which are not listed as "in progress", and for which all dependency tasks have been satisfied
all dependency tasks have been satisfied = NOT EXISTS ( join(dependencyTasks, task_states on task/version and where item_id) where is_candidate is null or true )
results (items which *should* be run, in order of preference)
all candidates (item + task_result) for which there is either:
1. no entry in task_states
2. an is_candidate=TRUE entry in task_states
3. an entry in task_states with mismatching task_version

@ -95,7 +95,13 @@ module.exports = function createKernel(configuration) {
: null, : null,
globalParallelize: (configuration.parallelTasks != null) globalParallelize: (configuration.parallelTasks != null)
? parallelize(configuration.parallelTasks) ? parallelize(configuration.parallelTasks)
: null : null,
taskDependencies: defaultValue(taskConfiguration.dependsOn, []).map((task) => {
return {
task: task,
taskVersion: defaultValue(configuration.tasks[task].taskVersion, "0")
};
})
}); });
return pipe([ return pipe([
@ -107,6 +113,9 @@ module.exports = function createKernel(configuration) {
} else { } else {
throw new Error(`Task '${task}' is defined to run for tags [${tags}], but no such task is defined`); throw new Error(`Task '${task}' is defined to run for tags [${tags}], but no such task is defined`);
} }
}).catch((error) => {
console.dir(error, { depth: null, colors: true });
throw error;
}); });
} }

@ -13,38 +13,68 @@ const pipe = require("@promistream/pipe");
const createMutationAPIWrapper = require("./mutation-api/wrapper"); const createMutationAPIWrapper = require("./mutation-api/wrapper");
let query = ` let query = `
WITH candidates AS ( WITH
dependency_tasks AS (
SELECT * FROM
json_to_recordset(:dependencyTaskDefinitions) AS x(task text, task_version text)
),
task_states AS (
SELECT
*,
(
is_successful = TRUE
AND (
expires_at < NOW()
OR is_invalidated = TRUE
)
) AS is_candidate
FROM task_results
),
matching_items AS (
SELECT
DISTINCT ON (items.id)
items.*,
states.updated_at AS result_date,
states.is_candidate,
states.task_version
FROM items
INNER JOIN tags
ON tags.item_id = items.id
AND tags.name = ANY(:tags)
LEFT JOIN task_states AS states
ON states.item_id = items.id
AND states.task = :task
WHERE
NOT EXISTS (
SELECT FROM tasks_in_progress AS pr WHERE pr.item_id = items.id
)
),
candidates AS (
SELECT * FROM matching_items
WHERE result_date IS NULL
UNION
SELECT * FROM matching_items
WHERE is_candidate = TRUE
OR NOT (task_version = :taskVersion)
)
(
SELECT SELECT
DISTINCT ON (items.id) *
items.*, FROM
results.expires_at, candidates
results.is_invalidated, WHERE
results.is_successful, NOT EXISTS (
results.updated_at AS result_date, SELECT
results.task_version task_states.*
FROM items FROM dependency_tasks
INNER JOIN tags LEFT JOIN task_states
ON tags.item_id = items.id ON dependency_tasks.task = task_states.task
AND tags.name = ANY(:tags) AND dependency_tasks.task_version = task_states.task_version
LEFT JOIN task_results AS results AND task_states.item_id = candidates.id
ON results.item_id = items.id WHERE
AND results.task = :task task_states.is_candidate IS NULL
WHERE NOT EXISTS ( OR task_states.is_candidate = TRUE
SELECT FROM tasks_in_progress AS pr WHERE pr.item_id = items.id
) )
)
(
SELECT * FROM candidates
WHERE result_date IS NULL
UNION
SELECT * FROM candidates
WHERE
is_successful = TRUE
AND (
expires_at < NOW()
OR is_invalidated = TRUE
OR NOT (task_version = :taskVersion)
)
) LIMIT :resultLimit; ) LIMIT :resultLimit;
`; `;
@ -57,7 +87,7 @@ module.exports = function (state) {
// FIXME: Transaction support! // FIXME: Transaction support!
return function createTaskStream({ task, taskVersion, tags, run, ttl, globalRateLimiter, globalParallelize }) { return function createTaskStream({ task, taskVersion, taskDependencies, tags, run, ttl, globalRateLimiter, globalParallelize }) {
// TODO: Make nicer // TODO: Make nicer
let ttlInSeconds = (ttl != null) let ttlInSeconds = (ttl != null)
? (typeof ttl === "number") ? (typeof ttl === "number")
@ -75,12 +105,13 @@ module.exports = function (state) {
tags: tags, tags: tags,
task: task, task: task,
taskVersion: taskVersion, taskVersion: taskVersion,
resultLimit: 1000 // TODO: Make configurable resultLimit: 1000, // TODO: Make configurable
dependencyTaskDefinitions: JSON.stringify(taskDependencies)
}); });
}).then((result) => { }).then((result) => {
let timeElapsed = Date.now() - startTime; let timeElapsed = Date.now() - startTime;
debug(`Task retrieval query took ${timeElapsed}ms and produced ${result.rowCount} results`); debug(`Task retrieval query for '${task}' took ${timeElapsed}ms and produced ${result.rowCount} results`);
if (result.rowCount > 0) { if (result.rowCount > 0) {
// console.log("rows:", result.rows); // console.log("rows:", result.rows);

Loading…
Cancel
Save