srap/src/queries.js

"use strict";

const Promise = require("bluebird");
// const { UniqueViolationError } = require("objection");
const dateFns = require("date-fns");

const { validateArguments } = require("@validatem/core");
const required = require("@validatem/required");
const requireEither = require("@validatem/require-either");
const isString = require("@validatem/is-string");
const isBoolean = require("@validatem/is-boolean");
const isFunction = require("@validatem/is-function");
const isNumber = require("@validatem/is-number");
const isDate = require("@validatem/is-date");
const arrayOf = require("@validatem/array-of");
const defaultTo = require("@validatem/default-to");
const anyProperty = require("@validatem/any-property");
const anything = require("@validatem/anything");
const ValidationError = require("@validatem/error");

const pipe = require("@promistream/pipe");
const combineSequentialStreaming = require("@promistream/combine-sequential-streaming");
const fromIterable = require("@promistream/from-iterable");
const fromNodeStream = require("@promistream/from-node-stream");
const createTypeTaggingStream = require("./streams/tag-type");

const { addSeconds } = require("date-fns");
const syncpipe = require("syncpipe");
const defaultValue = require("default-value");

function isTX(value) {
	if (value.where == null || value.raw == null) {
		throw new ValidationError(`Must be a valid Knex or Knex transaction instance`);
	}
}

function noop() {}

function taskResultsToObject(taskResults) {
	return syncpipe(taskResults, [
		(_) => _.map((result) => [ result.taskName, result.metadata ]),
		(_) => Object.fromEntries(_)
	]);
}

module.exports = function ({ db, knex }) {
	return {
		// FIXME: Make object API instead
		getItem: function (_tx, _id, _optional) {
			let [ tx, id, optional ] = validateArguments(arguments, {
				tx: [ required, isTX ],
				id: [ required, isString ],
				optional: [ defaultTo(false), isBoolean ]
			});

			return Promise.try(() => {
				return db.Alias.relatedQuery("item", tx)
					.for(id)
					.withGraphFetched("taskResults");
			}).then((results) => {
				if (optional === true || results.length > 0) {
					return results[0];
				} else {
					throw new Error(`No item exists with ID '${id}'`);
				}
			});
		},
		createItem: function (_tx, _options) {
			// NOTE: Using `update` instead of `data` makes it an upsert!
			// FIXME: Make failIfExists actually work, currently it does nothing as the UNIQUE constraint violation cannot occur for an upsert
			let [ tx, { id, tags, aliases, data, update, failIfExists, allowUpsert, parentID }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					id: [ required, isString ],
					tags: [ defaultTo([]), arrayOf(isString) ],
					aliases: [ defaultTo([]), arrayOf(isString) ],
					data: [ anything ], // FIXME: Check for object
					update: [ isFunction ],
					failIfExists: [ defaultTo(false), isBoolean ],
					allowUpsert: [ defaultTo(true), isBoolean ],
					parentID: [ isString ]
				}, requireEither([ "data", "update" ]) ]
			});

			// FIXME: Ensure that we run the transaction in full isolation mode, and retry in case of a conflict

			return Promise.try(() => {
				// NOTE: We look up by alias, since this is an upsert - and so if the specified ID already exists as an alias, we should update the existing item instead of creating a new one with the specified (aliased) ID
				return db.Alias
					.relatedQuery("item", tx)
					.for(id);
			}).then((existingItems) => {
				let existingItem = existingItems[0];
				
				let actualID = (existingItem != null)
					? existingItem.id
					: id;

				let existingData = (existingItem != null)
					? existingItem.data
					: {};

				let newData = (update != null)
					? update(existingData)
					: { ... existingData, ... data };

				// Make sure to add a self:self alias
				let allAliases = aliases.concat([ actualID ]);

				let newItem = {
					id: actualID,
					data: newData,
					createdBy: parentID,
					tags: tags.map((tag) => ({ name: tag })),
					aliases: allAliases.map((alias) => ({ alias: alias })),
					updatedAt: new Date()
				};

				if (allowUpsert) {
					// NOTE: We *always* do upserts here, even if the user specified `data` rather than `update`, because tags and aliases should always be added even if the item itself already exists. We trust the user not to accidentally reuse IDs between different kinds of objects (which would break in various other ways anyway).
					return db.Item.query(tx).upsertGraph(newItem, {
						insertMissing: true,
						noDelete: true
					});
				} else {
					return db.Item.query(tx).insertGraph(newItem, {
						insertMissing: true
					});
				}
			}).catch({ name: "UniqueViolationError", table: "srap_items" }, (error) => {
				if (failIfExists) {
					throw error;
				} else {
					// Do nothing, just ignore the failure
				}
			});
		},
		renameItem: function (_tx, _options) {
			// options == to || { from, to }
			let [ tx, { to, from }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					to: [ required, isString ],
					from: [ required, isString ]
				}]
			});

			return Promise.all([
				db.Item.query(tx).findById(from).patch({ id: to }),
				this.createAlias(tx, { from: to, to: to })
			]);
		},
		repointAliases: function (_tx, _options) {
			// { from, to }
			let [ tx, { to, from }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					to: [ required, isString ],
					from: [ required, isString ]
				}]
			});

			return db.Alias.query(tx)
				.patch({ itemId: to, updatedAt: new Date() })
				.where({ itemId: from });
		},
		mergeItem: function (_tx, _options) {
			// options = { from, into, merge, mergeMetadata{} }
			let [ tx, { from, into, merge, mergeMetadata }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					from: [ required, isString ],
					into: [ required, isString ],
					merge: [ required, isFunction ],
					mergeMetadata: [ defaultTo({}), anyProperty({
						key: [ required ],
						value: [ required, isFunction ]
					})],
				}]
			});

			return Promise.all([
				this.getItem(tx, from, true),
				this.getItem(tx, into, true),
			]).then(([ fromObj, intoObj ]) => {
				if (fromObj != null) {
					let defaultedIntoObj = defaultValue(intoObj, {
						id: into,
						data: {},
						taskResults: []
					});

					let newData = merge(defaultedIntoObj.data, fromObj.data);

					let fromTaskResults = taskResultsToObject(fromObj.taskResults);
					let intoTaskResults = taskResultsToObject(defaultedIntoObj.taskResults);

					// FIXME: Deduplicate function
					let allTaskKeys = Array.from(new Set([
						... Object.keys(fromTaskResults),
						... Object.keys(intoTaskResults)
					]));

					function selectNewestResult(taskA, taskB) {
						if (taskA == null) {
							return taskB;
						} else if (taskB == null) {
							return taskA;
						} else if (taskA.updatedAt > taskB.updatedAt) {
							return taskA;
						} else {
							return taskB;
						}
					}

					// TODO: Use merge-by-template here instead?

					let newTaskResults = allTaskKeys.map((key) => {
						let merger = mergeMetadata[key];
						let fromTask = fromTaskResults[key];
						let intoTask = intoTaskResults[key];

						if (merger != null) {
							// Generate a new TaskResult that includes data combined from both
							let newMetadata = merger(
								defaultValue(intoTask.metadata, {}),
								defaultValue(fromTask.metadata, {})
							);
							
							return {
								... intoTask,
								metadata: newMetadata,
								updatedAt: Date.now()
							};
						} else {
							// Take the newest known TaskResult and just make sure that it is pointing at the correct ID
							return {
								... selectNewestResult(intoTask, fromTask),
								itemId: defaultedIntoObj.id
							};
						}
					});

					let upsertOptions = {
						insertMissing: true,
						noDelete: true
					};

					return Promise.try(() => {
						// NOTE: Cannot use into.$query here because that adds an implicit query builder operation, which upsertGraph does not allow
						return db.Item.query(tx).upsertGraph({
							id: defaultedIntoObj.id,
							data: newData,
							taskResults: newTaskResults
						}, upsertOptions);
					}).then(() => {
						// NOTE: Repointing aliases has the side-effect of leaving a redirect from the source to the destination item, as each item has a self:self alias
						return this.repointAliases(tx, { from: fromObj.id, to: intoObj.id });
					}).then(() => {
						// NOTE: We don't use this.deleteItem, to sidestep any alias lookups
						return db.Item.query(tx).findById(fromObj.id).delete();
					});
				}
			});
		},
		deleteItem: function (_tx, _options) {
			// options = none || { id }
			let [ tx, { id }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					id: [ required, isString ]
				}]
			});

			return db.Alias.relatedQuery("item", tx)
				.for(id)
				.delete();
			// return db.Item.query(tx).findById(id).delete();
		},
		createAlias: function (_tx, _options) {
			// options = { from, to, failIfExists }
			let [ tx, { from, to, failIfExists }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					from: [ required, isString ],
					to: [ required, isString ],
					failIfExists: [ defaultTo(false), isBoolean ] // TODO: Shouldn't this default to true, for any occurrence outside of a merge/rename?
				}]
			});

			// Isolate this operation into a savepoint so that it can fail without breaking the entire transaction
			let promise = tx.transaction((tx) => {
				return db.Alias.query(tx).insert({
					alias: from,
					itemId: to,
					updatedAt: new Date()
				});
			});

			if (failIfExists) {
				return promise;
			} else {
				return Promise.resolve(promise)
					.catch({ name: "UniqueViolationError" }, noop);
			}
		},
		deleteAlias: function (_tx, _options) {
			let [ tx, { from }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					from: [ required, isString ]
				}]
			});

			// TODO: This cannot yet be propagated to the update feed, because we don't keep a record of deletions
			return db.Alias.query(tx).findById(from).delete();
		},
		updateData: function (_tx, _options) {
			// options = update || { id, update }
			let [ tx, { id, update }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					id: [ required, isString ],
					update: [ required, isFunction ]
				}]
			});

			// TODO: Figure out the proper delineation between 'creating' and 'updating' an item
			return this.createItem(tx, { id, update });
		},
		updateMetadata: function (_tx, _options) {
			// options = update || { id, update, taskName }
			let [ tx, { id, update, taskName, taskVersion, ttl }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					id: [ required, isString ],
					update: [ required, isFunction ],
					taskName: [ required, isString ],
					taskVersion: [ required, isString ],
					ttl: [ isNumber ]
				}]
			});
			// TODO: failIfExists

			// FIXME: metadata_updated_at
			return Promise.try(() => {
				return db.Alias.query(tx).findById(id);
			}).then((alias) => {
				let sharedFields = {
					isSuccessful: true,
					isInvalidated: false,
					taskVersion: taskVersion,
					updatedAt: new Date(),
					expiresAt: (ttl != null)
						? addSeconds(new Date(), ttl)
						: undefined
				};

				if (alias != null) {
					return Promise.try(() => {
						return db.TaskResult.query(tx).findById([ taskName, alias.itemId ]);
					}).then((taskResult) => {
						if (taskResult != null) {
							return taskResult.$query(tx).patch({
								... sharedFields,
								metadata: update(taskResult.metadata),
							});
						} else {
							return db.TaskResult.query(tx).insert({
								... sharedFields,
								task: taskName,
								itemId: id,
								metadata: update({})
							});
						}
					});
				}
			});
		},
		expire: function (_tx, _options) {
			// options = none || { id, taskName }
			let [ tx, { id, taskName }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [{
					id: [ required, isString ],
					taskName: [ required, isString ]
				}]
			});

			return Promise.try(() => {
				return db.Alias.query(tx).findById(id);
			}).then((alias) => {
				return db.TaskResult.query(tx)
					.where({ task: taskName, itemId: alias.itemId })
					.patch({ isInvalidated: true });
			});
		},
		setTTL: function (options) {
			// options = ttl || { id, taskName, ttl }
			// FIXME
		},
		allowFailure: function (allowed) {
			
		},
		log: function (category, message) {
			
		},
		countLockedTasks: function (tx) {
			return Promise.try(() => {
				return db.TaskInProgress.query(tx).count({ count: "*" });
			}).then((result) => {
				return result[0].count;
			});
		},
		getUpdates: function (_tx, _options) {
			// NOTE: This returns snake_cased keys! As we're bypassing the Objection internals, no casemapping occurs.
			let [ tx, { timestamp, prefix }] = validateArguments(arguments, {
				tx: [ required, isTX ],
				options: [ defaultTo({}), {
					timestamp: [ isDate ],
					prefix: [ isString ]
				}]
			});

			// NOTE: This is a hacky workaround - if we don't do this, then for some reason also entries *at* the exact timestamp are included, which is not what we want.
			// FIXME: Verify that this doesn't break anything, eg. when an entry is created inbetween the original timestamp and +1ms.
			let actualTimestamp = (timestamp != null)
				? dateFns.addMilliseconds(timestamp, 1)
				: undefined;

			function applyWhereClauses(query, idField) {
				if (timestamp != null) {
					// FIXME: An error in the query here throws an error, resulting in an abort handling bug in a promistream
					query = query.whereRaw(`updated_at > ?`, [ actualTimestamp ]);
				}

				if (prefix != null) {
					query = query.whereRaw(`${idField} LIKE ?`, [ `${prefix.replace(/%/g, "\\%")}%` ]);
				}

				return query;
			}

			function* streamGenerator() {
				yield pipe([
					fromNodeStream.fromReadable(
						applyWhereClauses(db.Item.query(tx), "id").toKnexQuery().stream()
					),
					createTypeTaggingStream("item")
				]);
				
				yield pipe([
					fromNodeStream.fromReadable(
						// NOTE: We are only interested in aliases which don't point at themselves
						applyWhereClauses(db.Alias.query(tx).where("alias", "!=", knex.ref("item_id")), "alias").toKnexQuery().stream()
					),
					createTypeTaggingStream("alias")
				]);

				yield pipe([
					fromNodeStream.fromReadable(
						applyWhereClauses(db.TaskResult.query(tx), "item_id").toKnexQuery().stream()
					),
					createTypeTaggingStream("taskResult")
				]);
			}

			return pipe([
				fromIterable(streamGenerator()),
				combineSequentialStreaming()
			]);
		}
	};
};