You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
54 lines
1.7 KiB
JavaScript
54 lines
1.7 KiB
JavaScript
"use strict";
|
|
|
|
const chalk = require("chalk");
|
|
const pipe = require("@promistream/pipe");
|
|
const filter = require("@promistream/filter");
|
|
const map = require("@promistream/map");
|
|
|
|
const logStatus = require("./log-status");
|
|
|
|
module.exports = function ({ backend }) {
|
|
function runTask(task, item) {
|
|
let queue = [];
|
|
let api = backend.forItem({ task: task, id: item.id, mutationQueue: queue });
|
|
|
|
return Promise.try(() => {
|
|
logStatus(task, chalk.bold.cyan, "started", item.id);
|
|
|
|
// NOTE: We only pass in the item data itself, *not* any associated metadata like tags. If the scraping task wants access to that sort of information, it should do a `getItem` call from within its task logic where needed.
|
|
// FIXME: Is that actually still true post-refactor?
|
|
task.run({
|
|
data: item.data,
|
|
... api.exposed
|
|
});
|
|
}).then(() => {
|
|
return backend.topLevel.runInTransaction((tx) => {
|
|
|
|
// FIXME: use queue
|
|
});
|
|
}).then(async () => {
|
|
await api.internal.markTaskCompleted();
|
|
logStatus(task, chalk.bold.green, "completed", item.id);
|
|
return { status: "completed", item: item };
|
|
}).catch(async (error) => {
|
|
await api.internal.markTaskFailed(null, { error });
|
|
logStatus(task, chalk.bold.red, "failed", `${item.id}: ${error.stack}`);
|
|
return { status: "failed", item: item, error: error };
|
|
});
|
|
}
|
|
|
|
return function createTaskKernelStream(task) {
|
|
return pipe([
|
|
backend.topLevel.getTaskStream(null, { task: task }),
|
|
filter((item) => backend.forItem({ task: task, id: item.id }).internal.lock()),
|
|
map((item) => {
|
|
return Promise.try(() => {
|
|
return runTask(task, item);
|
|
}).tap(() => {
|
|
return backend.forItem({ task: task, id: item.id }).internal.unlock();
|
|
});
|
|
})
|
|
]);
|
|
};
|
|
};
|