You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
475 lines
14 KiB
JavaScript
475 lines
14 KiB
JavaScript
"use strict";
|
|
|
|
const Promise = require("bluebird");
|
|
// const { UniqueViolationError } = require("objection");
|
|
const dateFns = require("date-fns");
|
|
|
|
const { validateArguments } = require("@validatem/core");
|
|
const required = require("@validatem/required");
|
|
const requireEither = require("@validatem/require-either");
|
|
const isString = require("@validatem/is-string");
|
|
const isBoolean = require("@validatem/is-boolean");
|
|
const isFunction = require("@validatem/is-function");
|
|
const isNumber = require("@validatem/is-number");
|
|
const isDate = require("@validatem/is-date");
|
|
const arrayOf = require("@validatem/array-of");
|
|
const defaultTo = require("@validatem/default-to");
|
|
const anyProperty = require("@validatem/any-property");
|
|
const anything = require("@validatem/anything");
|
|
const ValidationError = require("@validatem/error");
|
|
|
|
const pipe = require("@promistream/pipe");
|
|
const combineSequentialStreaming = require("@promistream/combine-sequential-streaming");
|
|
const fromIterable = require("@promistream/from-iterable");
|
|
const fromNodeStream = require("@promistream/from-node-stream");
|
|
const createTypeTaggingStream = require("./streams/tag-type");
|
|
|
|
const { addSeconds } = require("date-fns");
|
|
const syncpipe = require("syncpipe");
|
|
const defaultValue = require("default-value");
|
|
|
|
function isTX(value) {
|
|
if (value.where == null || value.raw == null) {
|
|
throw new ValidationError(`Must be a valid Knex or Knex transaction instance`);
|
|
}
|
|
}
|
|
|
|
function noop() {}
|
|
|
|
function taskResultsToObject(taskResults) {
|
|
return syncpipe(taskResults, [
|
|
(_) => _.map((result) => [ result.taskName, result.metadata ]),
|
|
(_) => Object.fromEntries(_)
|
|
]);
|
|
}
|
|
|
|
module.exports = function ({ db, knex }) {
|
|
return {
|
|
// FIXME: Make object API instead
|
|
getItem: function (_tx, _id, _optional) {
|
|
let [ tx, id, optional ] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
id: [ required, isString ],
|
|
optional: [ defaultTo(false), isBoolean ]
|
|
});
|
|
|
|
return Promise.try(() => {
|
|
return db.Alias.relatedQuery("item", tx)
|
|
.for(id)
|
|
.withGraphFetched("taskResults");
|
|
}).then((results) => {
|
|
if (optional === true || results.length > 0) {
|
|
return results[0];
|
|
} else {
|
|
throw new Error(`No item exists with ID '${id}'`);
|
|
}
|
|
});
|
|
},
|
|
createItem: function (_tx, _options) {
|
|
// NOTE: Using `update` instead of `data` makes it an upsert!
|
|
// FIXME: Make failIfExists actually work, currently it does nothing as the UNIQUE constraint violation cannot occur for an upsert
|
|
let [ tx, { id, tags, aliases, data, update, failIfExists, allowUpsert, parentID }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
id: [ required, isString ],
|
|
tags: [ defaultTo([]), arrayOf(isString) ],
|
|
aliases: [ defaultTo([]), arrayOf(isString) ],
|
|
data: [ anything ], // FIXME: Check for object
|
|
update: [ isFunction ],
|
|
failIfExists: [ defaultTo(false), isBoolean ],
|
|
allowUpsert: [ defaultTo(true), isBoolean ],
|
|
parentID: [ isString ]
|
|
}, requireEither([ "data", "update" ]) ]
|
|
});
|
|
|
|
// FIXME: Ensure that we run the transaction in full isolation mode, and retry in case of a conflict
|
|
|
|
return Promise.try(() => {
|
|
// NOTE: We look up by alias, since this is an upsert - and so if the specified ID already exists as an alias, we should update the existing item instead of creating a new one with the specified (aliased) ID
|
|
return db.Alias
|
|
.relatedQuery("item", tx)
|
|
.for(id);
|
|
}).then((existingItems) => {
|
|
let existingItem = existingItems[0];
|
|
|
|
let actualID = (existingItem != null)
|
|
? existingItem.id
|
|
: id;
|
|
|
|
let existingData = (existingItem != null)
|
|
? existingItem.data
|
|
: {};
|
|
|
|
let newData = (update != null)
|
|
? update(existingData)
|
|
: { ... existingData, ... data };
|
|
|
|
// Make sure to add a self:self alias
|
|
let allAliases = aliases.concat([ actualID ]);
|
|
|
|
let newItem = {
|
|
id: actualID,
|
|
data: newData,
|
|
createdBy: parentID,
|
|
tags: tags.map((tag) => ({ name: tag })),
|
|
aliases: allAliases.map((alias) => ({ alias: alias })),
|
|
updatedAt: new Date()
|
|
};
|
|
|
|
if (allowUpsert) {
|
|
// NOTE: We *always* do upserts here, even if the user specified `data` rather than `update`, because tags and aliases should always be added even if the item itself already exists. We trust the user not to accidentally reuse IDs between different kinds of objects (which would break in various other ways anyway).
|
|
return db.Item.query(tx).upsertGraph(newItem, {
|
|
insertMissing: true,
|
|
noDelete: true
|
|
});
|
|
} else {
|
|
return db.Item.query(tx).insertGraph(newItem, {
|
|
insertMissing: true
|
|
});
|
|
}
|
|
}).catch({ name: "UniqueViolationError", table: "srap_items" }, (error) => {
|
|
if (failIfExists) {
|
|
throw error;
|
|
} else {
|
|
// Do nothing, just ignore the failure
|
|
}
|
|
});
|
|
},
|
|
renameItem: function (_tx, _options) {
|
|
// options == to || { from, to }
|
|
let [ tx, { to, from }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
to: [ required, isString ],
|
|
from: [ required, isString ]
|
|
}]
|
|
});
|
|
|
|
return Promise.all([
|
|
db.Item.query(tx).findById(from).patch({ id: to }),
|
|
this.createAlias(tx, { from: to, to: to })
|
|
]);
|
|
},
|
|
repointAliases: function (_tx, _options) {
|
|
// { from, to }
|
|
let [ tx, { to, from }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
to: [ required, isString ],
|
|
from: [ required, isString ]
|
|
}]
|
|
});
|
|
|
|
return db.Alias.query(tx)
|
|
.patch({ itemId: to, updatedAt: new Date() })
|
|
.where({ itemId: from });
|
|
},
|
|
mergeItem: function (_tx, _options) {
|
|
// options = { from, into, merge, mergeMetadata{} }
|
|
let [ tx, { from, into, merge, mergeMetadata }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
from: [ required, isString ],
|
|
into: [ required, isString ],
|
|
merge: [ required, isFunction ],
|
|
mergeMetadata: [ defaultTo({}), anyProperty({
|
|
key: [ required ],
|
|
value: [ required, isFunction ]
|
|
})],
|
|
}]
|
|
});
|
|
|
|
return Promise.all([
|
|
this.getItem(tx, from, true),
|
|
this.getItem(tx, into, true),
|
|
]).then(([ fromObj, intoObj ]) => {
|
|
if (fromObj != null) {
|
|
let defaultedIntoObj = defaultValue(intoObj, {
|
|
id: into,
|
|
data: {},
|
|
taskResults: []
|
|
});
|
|
|
|
let newData = merge(defaultedIntoObj.data, fromObj.data);
|
|
|
|
let fromTaskResults = taskResultsToObject(fromObj.taskResults);
|
|
let intoTaskResults = taskResultsToObject(defaultedIntoObj.taskResults);
|
|
|
|
// FIXME: Deduplicate function
|
|
let allTaskKeys = Array.from(new Set([
|
|
... Object.keys(fromTaskResults),
|
|
... Object.keys(intoTaskResults)
|
|
]));
|
|
|
|
function selectNewestResult(taskA, taskB) {
|
|
if (taskA == null) {
|
|
return taskB;
|
|
} else if (taskB == null) {
|
|
return taskA;
|
|
} else if (taskA.updatedAt > taskB.updatedAt) {
|
|
return taskA;
|
|
} else {
|
|
return taskB;
|
|
}
|
|
}
|
|
|
|
// TODO: Use merge-by-template here instead?
|
|
|
|
let newTaskResults = allTaskKeys.map((key) => {
|
|
let merger = mergeMetadata[key];
|
|
let fromTask = fromTaskResults[key];
|
|
let intoTask = intoTaskResults[key];
|
|
|
|
if (merger != null) {
|
|
// Generate a new TaskResult that includes data combined from both
|
|
let newMetadata = merger(
|
|
defaultValue(intoTask.metadata, {}),
|
|
defaultValue(fromTask.metadata, {})
|
|
);
|
|
|
|
return {
|
|
... intoTask,
|
|
metadata: newMetadata,
|
|
updatedAt: Date.now()
|
|
};
|
|
} else {
|
|
// Take the newest known TaskResult and just make sure that it is pointing at the correct ID
|
|
return {
|
|
... selectNewestResult(intoTask, fromTask),
|
|
itemId: defaultedIntoObj.id
|
|
};
|
|
}
|
|
});
|
|
|
|
let upsertOptions = {
|
|
insertMissing: true,
|
|
noDelete: true
|
|
};
|
|
|
|
return Promise.try(() => {
|
|
// NOTE: Cannot use into.$query here because that adds an implicit query builder operation, which upsertGraph does not allow
|
|
return db.Item.query(tx).upsertGraph({
|
|
id: defaultedIntoObj.id,
|
|
data: newData,
|
|
taskResults: newTaskResults
|
|
}, upsertOptions);
|
|
}).then(() => {
|
|
// NOTE: Repointing aliases has the side-effect of leaving a redirect from the source to the destination item, as each item has a self:self alias
|
|
return this.repointAliases(tx, { from: fromObj.id, to: intoObj.id });
|
|
}).then(() => {
|
|
// NOTE: We don't use this.deleteItem, to sidestep any alias lookups
|
|
return db.Item.query(tx).findById(fromObj.id).delete();
|
|
});
|
|
}
|
|
});
|
|
},
|
|
deleteItem: function (_tx, _options) {
|
|
// options = none || { id }
|
|
let [ tx, { id }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
id: [ required, isString ]
|
|
}]
|
|
});
|
|
|
|
return db.Alias.relatedQuery("item", tx)
|
|
.for(id)
|
|
.delete();
|
|
// return db.Item.query(tx).findById(id).delete();
|
|
},
|
|
createAlias: function (_tx, _options) {
|
|
// options = { from, to, failIfExists }
|
|
let [ tx, { from, to, failIfExists }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
from: [ required, isString ],
|
|
to: [ required, isString ],
|
|
failIfExists: [ defaultTo(false), isBoolean ] // TODO: Shouldn't this default to true, for any occurrence outside of a merge/rename?
|
|
}]
|
|
});
|
|
|
|
// Isolate this operation into a savepoint so that it can fail without breaking the entire transaction
|
|
let promise = tx.transaction((tx) => {
|
|
return db.Alias.query(tx).insert({
|
|
alias: from,
|
|
itemId: to,
|
|
updatedAt: new Date()
|
|
});
|
|
});
|
|
|
|
if (failIfExists) {
|
|
return promise;
|
|
} else {
|
|
return Promise.resolve(promise)
|
|
.catch({ name: "UniqueViolationError" }, noop);
|
|
}
|
|
},
|
|
deleteAlias: function (_tx, _options) {
|
|
let [ tx, { from }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
from: [ required, isString ]
|
|
}]
|
|
});
|
|
|
|
// TODO: This cannot yet be propagated to the update feed, because we don't keep a record of deletions
|
|
return db.Alias.query(tx).findById(from).delete();
|
|
},
|
|
updateData: function (_tx, _options) {
|
|
// options = update || { id, update }
|
|
let [ tx, { id, update }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
id: [ required, isString ],
|
|
update: [ required, isFunction ]
|
|
}]
|
|
});
|
|
|
|
// TODO: Figure out the proper delineation between 'creating' and 'updating' an item
|
|
return this.createItem(tx, { id, update });
|
|
},
|
|
updateMetadata: function (_tx, _options) {
|
|
// options = update || { id, update, taskName }
|
|
let [ tx, { id, update, taskName, taskVersion, ttl }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
id: [ required, isString ],
|
|
update: [ required, isFunction ],
|
|
taskName: [ required, isString ],
|
|
taskVersion: [ required, isString ],
|
|
ttl: [ isNumber ]
|
|
}]
|
|
});
|
|
// TODO: failIfExists
|
|
|
|
// FIXME: metadata_updated_at
|
|
return Promise.try(() => {
|
|
return db.Alias.query(tx).findById(id);
|
|
}).then((alias) => {
|
|
let sharedFields = {
|
|
isSuccessful: true,
|
|
isInvalidated: false,
|
|
taskVersion: taskVersion,
|
|
updatedAt: new Date(),
|
|
expiresAt: (ttl != null)
|
|
? addSeconds(new Date(), ttl)
|
|
: undefined
|
|
};
|
|
|
|
if (alias != null) {
|
|
return Promise.try(() => {
|
|
return db.TaskResult.query(tx).findById([ taskName, alias.itemId ]);
|
|
}).then((taskResult) => {
|
|
if (taskResult != null) {
|
|
return taskResult.$query(tx).patch({
|
|
... sharedFields,
|
|
metadata: update(taskResult.metadata),
|
|
});
|
|
} else {
|
|
return db.TaskResult.query(tx).insert({
|
|
... sharedFields,
|
|
task: taskName,
|
|
itemId: id,
|
|
metadata: update({})
|
|
});
|
|
}
|
|
});
|
|
}
|
|
});
|
|
},
|
|
expire: function (_tx, _options) {
|
|
// options = none || { id, taskName }
|
|
let [ tx, { id, taskName }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [{
|
|
id: [ required, isString ],
|
|
taskName: [ required, isString ]
|
|
}]
|
|
});
|
|
|
|
return Promise.try(() => {
|
|
return db.Alias.query(tx).findById(id);
|
|
}).then((alias) => {
|
|
return db.TaskResult.query(tx)
|
|
.where({ task: taskName, itemId: alias.itemId })
|
|
.patch({ isInvalidated: true });
|
|
});
|
|
},
|
|
setTTL: function (options) {
|
|
// options = ttl || { id, taskName, ttl }
|
|
// FIXME
|
|
},
|
|
allowFailure: function (allowed) {
|
|
|
|
},
|
|
log: function (category, message) {
|
|
|
|
},
|
|
countLockedTasks: function (tx) {
|
|
return Promise.try(() => {
|
|
return db.TaskInProgress.query(tx).count({ count: "*" });
|
|
}).then((result) => {
|
|
return result[0].count;
|
|
});
|
|
},
|
|
getUpdates: function (_tx, _options) {
|
|
// NOTE: This returns snake_cased keys! As we're bypassing the Objection internals, no casemapping occurs.
|
|
let [ tx, { timestamp, prefix }] = validateArguments(arguments, {
|
|
tx: [ required, isTX ],
|
|
options: [ defaultTo({}), {
|
|
timestamp: [ isDate ],
|
|
prefix: [ isString ]
|
|
}]
|
|
});
|
|
|
|
// NOTE: This is a hacky workaround - if we don't do this, then for some reason also entries *at* the exact timestamp are included, which is not what we want.
|
|
// FIXME: Verify that this doesn't break anything, eg. when an entry is created inbetween the original timestamp and +1ms.
|
|
let actualTimestamp = (timestamp != null)
|
|
? dateFns.addMilliseconds(timestamp, 1)
|
|
: undefined;
|
|
|
|
function applyWhereClauses(query, idField) {
|
|
if (timestamp != null) {
|
|
// FIXME: An error in the query here throws an error, resulting in an abort handling bug in a promistream
|
|
query = query.whereRaw(`updated_at > ?`, [ actualTimestamp ]);
|
|
}
|
|
|
|
if (prefix != null) {
|
|
query = query.whereRaw(`${idField} LIKE ?`, [ `${prefix.replace(/%/g, "\\%")}%` ]);
|
|
}
|
|
|
|
return query;
|
|
}
|
|
|
|
function* streamGenerator() {
|
|
yield pipe([
|
|
fromNodeStream.fromReadable(
|
|
applyWhereClauses(db.Item.query(tx), "id").toKnexQuery().stream()
|
|
),
|
|
createTypeTaggingStream("item")
|
|
]);
|
|
|
|
yield pipe([
|
|
fromNodeStream.fromReadable(
|
|
// NOTE: We are only interested in aliases which don't point at themselves
|
|
applyWhereClauses(db.Alias.query(tx).where("alias", "!=", knex.ref("item_id")), "alias").toKnexQuery().stream()
|
|
),
|
|
createTypeTaggingStream("alias")
|
|
]);
|
|
|
|
yield pipe([
|
|
fromNodeStream.fromReadable(
|
|
applyWhereClauses(db.TaskResult.query(tx), "item_id").toKnexQuery().stream()
|
|
),
|
|
createTypeTaggingStream("taskResult")
|
|
]);
|
|
}
|
|
|
|
return pipe([
|
|
fromIterable(streamGenerator()),
|
|
combineSequentialStreaming()
|
|
]);
|
|
}
|
|
};
|
|
};
|