"use strict"; const Promise = require("bluebird"); const defaultValue = require("default-value"); const chalk = require("chalk"); const util = require("util"); const syncpipe = require("syncpipe"); const rateLimit = require("@ppstreams/rate-limit"); const simpleSink = require("@promistream/simple-sink"); const pipe = require("@promistream/pipe"); const parallelize = require("@ppstreams/parallelize"); const initialize = require("./initialize"); // TODO: Publish this as a separate package // Inverts an object of arrays, eg. {a: [x, y], b: [x, z]} becomes {x: [a, b], y: [a], z: [b]} // Useful for eg. tag mappings function invertMapping(object) { let newObject = {}; for (let [ key, valueList ] of Object.entries(object)) { for (let value of valueList) { if (newObject[value] == null) { newObject[value] = []; } newObject[value].push(key); } } return newObject; } function log(value) { console.log(value); return value; } module.exports = function createKernel(configuration) { return Promise.try(() => { return initialize({ knexfile: { client: "pg", connection: configuration.database } }); }).then((state) => { const queries = require("./queries")(state); const createTaskStream = require("./task-stream")(state); let { knex } = state; function insertSeeds() { return Promise.map(configuration.seed, (item) => { return queries.createItem(knex, { ... item, allowUpsert: false, failIfExists: false }); }); } function checkLockedTasks() { return Promise.try(() => { return queries.countLockedTasks(knex); }).then((lockedCount) => { if (lockedCount > 0) { console.log(`${chalk.bold.red("WARNING:")} There are ${lockedCount} tasks currently locked, and they will not be run! This may be caused by a process crash in the past. See the documentation for more details on how to solve this issue.`); } }); } function runTaskStreams() { let tasks = invertMapping(configuration.tags); let attachToGlobalRateLimit = (configuration.taskInterval != null) ? rateLimit.clonable(configuration.taskInterval) : undefined; console.log(`Starting ${Object.keys(tasks).length} tasks...`); return Promise.map(Object.entries(tasks), ([ task, tags ]) => { let taskConfiguration = configuration.tasks[task]; if (taskConfiguration != null) { let taskStream = createTaskStream({ task: task, tags: tags, taskVersion: defaultValue(taskConfiguration.version, "0"), ttl: taskConfiguration.ttl, run: taskConfiguration.run, globalRateLimiter: (attachToGlobalRateLimit != null) ? attachToGlobalRateLimit() : null, globalParallelize: (configuration.parallelTasks != null) ? parallelize(configuration.parallelTasks) : null, taskDependencies: defaultValue(taskConfiguration.dependsOn, []).map((task) => { return { task: task, taskVersion: defaultValue(configuration.tasks[task].taskVersion, "0") }; }) }); return pipe([ taskStream, simpleSink((completedItem) => { console.log(`[completed] ${completedItem.id}`); }) ]).read(); } else { throw new Error(`Task '${task}' is defined to run for tags [${tags}], but no such task is defined`); } }).catch((error) => { console.dir(error, { depth: null, colors: true }); throw error; }); } function simulateTask(id, task) { let taskConfiguration = configuration.tasks[task]; let methods = [ "createItem", "renameItem", "mergeItem", "deleteItem", "createAlias", "deleteAlias", "updateData", "updateMetadata", "expire" ]; let simulatedMethods = syncpipe(methods, [ (_) => _.map((method) => [ method, function() { console.log(`${chalk.bold.yellow.bgBlack(`${method} (simulated):`)} ${util.inspect(arguments, { colors: true, depth: null })}`); }]), (_) => Object.fromEntries(_) ]); return Promise.try(() => { return queries.getItem(knex, id); }).then((item) => { return taskConfiguration.run({ id: item.id, data: item.data, getItem: function (id) { return queries.getItem(knex, id); }, ... simulatedMethods }); }); } return { run: function runKernel() { return Promise.try(() => { return insertSeeds(); }).then(() => { return checkLockedTasks(); }).then(() => { return runTaskStreams(); }); }, simulate: function simulate({ itemID, task }) { return Promise.try(() => { return insertSeeds(); }).then(() => { return checkLockedTasks(); }).then(() => { return simulateTask(itemID, task); }); }, shutdown: function () { // TODO: Properly lock all public methods after shutdown is called, and wait for any running tasks to have completed knex.destroy(); } }; }); };