dlayer: Add module system

src/packages/dlayer/deep-merge.js

@@ -0,0 +1,38 @@
+"use strict";
+
+// A strict deep-merging implementation that *only* merges regular objects, and prevents prototype pollution
+
+function isObject(value) {
+	// TODO: Disallow special object types, for statically defined values (or just disallow specifying static values in the root schema in dlayer?)
+	return (value != null && typeof value === "object" && !Array.isArray(value));
+}
+
+module.exports = function deepMerge(a, b) {
+	let merged = Object.create(null);
+	let keys = new Set([ ... Object.keys(a), ... Object.keys(b) ]);
+
+	for (let key of keys) {
+		// Technically over-blocks *any* 'constructor' key
+		if (key === "__proto__" || key === "constructor") {
+			continue;
+		}
+
+		let valueA = a[key];
+		let valueB = b[key];
+
+		if (isObject(valueA) && valueB === undefined) {
+			merged[key] = valueA;
+		} else if (isObject(valueB) && valueA === undefined) {
+			merged[key] = valueB;
+		} else if (isObject(valueA) && isObject(valueB)) {
+			merged[key] = deepMerge(valueA, valueB);
+		} else if (!isObject(valueA) && !isObject(valueB)) {
+			merged[key] = valueB ?? valueA;
+		} else {
+			// FIXME: Identifiable error type, and include the error path as well
+			throw new Error("Cannot merge non-object into object");
+		}
+	}
+
+	return merged;
+};

src/packages/dlayer/docs.md

@@ -1,3 +1,5 @@
+A module system for your data sources.
+
 ```js
 {
 	system: {

src/packages/dlayer/index.js

@@ -6,16 +6,18 @@ const syncpipe = require("syncpipe");
 
 const Result = require("../result");
 const createCursor = require("./cursor");
+const deepMerge = require("./deep-merge");
+const loadModules = require("./load-modules");
 
 // TODO: Bounded/unbounded recursion
-// TODO: context
+// TODO: Should we allow async context generation? Both in root schema *and* in modules
 // TODO: $required query predicate
 // TODO: Lazy query objects, which get initialized by calling a function that gets the parent object as an argument? This would not be serializable over the network!
 // FIXME: $getProperty, $getPropertyPath, maybe $resolveObject/$query?
 // FIXME: Allow setting an evaluation depth limit for queries, to limit eg. recursion
 // FIXME: recurseDepth, recurseLabel/recurseGoto
 
-/* Data structure design:
+/* Process design:
 
 - The process starts with:
 	1. A query tree, a nested object representing the query from the user
@@ -97,10 +99,8 @@ function analyzeSubquery(subquery) {
 }
 
 function makeInstruction(cursor, queryKey) {
-	// let schemaKey = cursor.childQuery(queryKey)?.$key ?? queryKey; // $key is for handling aliases
 	let childCursor = cursor.child(queryKey);
 	let handler = childCursor.schema ?? cursor.schema.$anyKey;
-	// TODO: Maybe clean up all the .child stuff here by moving the `$key` logic into the cursor implementation instead, as it seems like nothing else in dlayer needs to care about the aliasing
 
 	return {
 		... analyzeSubquery(childCursor.query),
@@ -119,7 +119,7 @@ function assignErrorPath(error, cursor) {
 
 // TODO: build a sample todo-list schema for testing out fetches, mutations, and combinations of them, including on collections
 
-function makeEnvironment(context) {
+function makeEnvironment(context, getContextForModule) {
 	function callHandler(instruction) {
 		// NOTE: cursor is assumed to already be the key of the child
 		let { schemaKey, handler, args, allowErrors, cursor } = instruction;
@@ -127,7 +127,13 @@ function makeEnvironment(context) {
 		if (handler != null) {
 			return Promise.try(() => {
 				// This calls the data provider in the schema
-				return Result.wrapAsync(() => maybeCall(handler, [ args, context ], cursor.parent.schema));
+				if (handler.__moduleID != null) {
+					// Defined in a module
+					return Result.wrapAsync(() => maybeCall(handler.func, [ args, getContextForModule(handler.__moduleID) ], cursor.parent.schema));
+				} else {
+					// Defined in the root schema
+					return Result.wrapAsync(() => maybeCall(handler, [ args, context ], cursor.parent.schema));
+				}
 			}).then((result) => {
 				if (result.isOK) {
 					return result.value();
@@ -220,6 +226,9 @@ function makeEnvironment(context) {
 module.exports = function createDLayer(options) {
 	// options = { schema, makeContext }
 
+	let loaded = loadModules(options.modules);
+	let schema = deepMerge(loaded.root, options.schema);
+
 	return {
 		query: function (query, context) {
 			let generatedContext = (options.makeContext != null)
@@ -227,6 +236,7 @@ module.exports = function createDLayer(options) {
 				: {};
 
 			function getProperty(object, property, args = {}) {
+				// TODO: Should this allow a single-argument, property-string-only variant for looking up properties on self?
 				// FIXME: Validatem
 				if (object == null) {
 					throw new Error(`Empty object passed`);
@@ -240,6 +250,29 @@ module.exports = function createDLayer(options) {
 				}
 			}
 
+			function make(typeID, args, existenceRequired) {
+				let type = loaded.types[typeID].func;
+
+				if (type == null) {
+					if (existenceRequired === true) {
+						throw new Error(`No type named '${typeID}' exists`);
+					} else {
+						return;
+					}
+				} else {
+					let instance = type(args);
+
+					if (loaded.extensions[typeID] != null) {
+						for (let [ key, extension ] of Object.entries(loaded.extensions[typeID])) {
+							// TODO: Possibly make this more performant by making it possible to do an Object.assign? Or does that not matter for performance?
+							instance[key] = extension.func;
+						}
+					}
+
+					return instance;
+				}
+			}
+
 			let combinedContext = {
 				... generatedContext,
 				... context,
@@ -258,15 +291,21 @@ module.exports = function createDLayer(options) {
 							return null;
 						}
 					}, object);
+				},
+				$make: function (typeID, args) {
+					return make(typeID, args, true);
+				},
+				$maybeMake: function (typeID, args) {
+					return make(typeID, args, false);
 				}
 			};
 			
 			let cursor = createCursor({
 				query: query,
-				schema: options.schema
+				schema: schema
 			});
 
-			let evaluate = makeEnvironment(combinedContext);
+			let evaluate = makeEnvironment(combinedContext, loaded.makeContextFactory(combinedContext));
 
 			// FIXME: Currently, top-level errors do not get a path property assigned to them, because that assignment happens on nested calls above
 			return evaluate(cursor);

src/packages/dlayer/load-modules.js

@@ -0,0 +1,128 @@
+"use strict";
+
+// const mergeByTemplate = require("merge-by-template");
+const deepMerge = require("./deep-merge");
+const syncpipe = require("syncpipe");
+
+/*
+Take a list of modules; each module specifies a name, schema root, types, type extensions, and a context factory function. Each module is internally assigned a unique ID. This unique ID is associated with each type factory and type extension method, and used as the key for a map of context factories; that way, upon invoking those methods, the module's own corresponding context can be injected. Only a single context should be created per module per request, so there should be a cache layer for the contexts (keyed by module ID), with each request creating a new cache.
+*/
+
+// NOTE: This can be global because we identify existing assignments by object identity, and that will never conflict
+let numberedModules = new WeakMap();
+let currentModuleNumber = 0;
+
+function getModuleID(module) {
+	if (!numberedModules.has(module)) {
+		numberedModules.set(module, currentModuleNumber++);
+	}
+	
+	return numberedModules.get(module);
+}
+
+function createTypeTracker() {
+	let typeFactories = {};
+
+	return {
+		add: function (module, name, factory) {
+			if (typeFactories[name] != null) {
+				let existingEntry = typeFactories[name];
+				throw new Error(`Type '${name}' already exists (from module '${module.name}', already defined by module '${existingEntry.source.name}')`);
+			} else {
+				typeFactories[name] = {
+					source: module,
+					// No context provided to type factory functions for now, since they are not allowed to be async for now anyway
+					// FIXME: Maybe add a warning if the user returns a Promise from a factory, asking them to file a bug if they really need it?
+					// func: wrapModuleFunction(module, factory)
+					func: factory
+				};
+			}
+		},
+		get: function () {
+			return typeFactories;
+		}
+	};
+}
+
+function createExtensionTracker() {
+	let extendedTypes = {};
+
+	return {
+		add: function (module, type, name, method) {
+			if (extendedTypes[type] == null) {
+				extendedTypes[type] = {};
+			}
+
+			let extensions = extendedTypes[type];
+
+			if (extensions[name] != null) {
+				let existingEntry = extensions[name];
+				throw new Error(`Type '${type}' already has a method extension named '${name}' (from module '${module.name}', already defined by module '${existingEntry.source.name}')`);
+			} else {
+				extensions[name] = {
+					source: module,
+					func: wrapModuleFunction(module, method)
+				};
+			}
+		},
+		get: function () {
+			return extendedTypes;
+		}
+	};
+}
+
+function wrapModuleFunction(module, func) {
+	return { __moduleID: getModuleID(module), func: func };
+}
+
+function defaultContext() {
+	// Fallback function that generates an empty context, for when a module doesn't specify a makeContext handler
+	return {};
+}
+
+module.exports = function (modules) {
+	// TODO: Eventually replace hand-crafted merging logic with merge-by-template, once it can support this usecase properly(tm)
+	// TODO: Fix merge-by-template so that reasonable error messages can be generated here, that are actually aware of eg. the conflicting key
+
+	let types = createTypeTracker();
+	let typeExtensions = createExtensionTracker();
+
+	let contextFactories = syncpipe(modules, [
+		_ => _.map((module) => [ getModuleID(module), module.makeContext ?? defaultContext ]),
+		_ => new Map(_)
+	]);
+
+	let schemaRoots = modules.map((module) => module.root ?? {});
+
+	for (let module of modules) {
+		for (let [ type, factory ] of Object.entries(module.types)) {
+			types.add(module, type, factory);
+		}
+
+		for (let [ type, extensions ] of Object.entries(module.extensions)) {
+			for (let [ name, method ] of Object.entries(extensions)) {
+				typeExtensions.add(module, type, name, method);
+			}
+		}
+	}
+
+	return {
+		root: schemaRoots.reduce(deepMerge, {}),
+		types: types.get(),
+		extensions: typeExtensions.get(),
+		makeContextFactory: function (baseContext) {
+			let cache = new Map();
+
+			return function makeContextForModule(moduleID) {
+				if (!cache.has(moduleID)) {
+					cache.set(moduleID, {
+						... baseContext,
+						... contextFactories.get(moduleID)()
+					});
+				}
+
+				return cache.get(moduleID);
+			};
+		}
+	};
+};

src/packages/dlayer/test-modules.js

@@ -0,0 +1,137 @@
+"use strict";
+
+const Promise = require("bluebird");
+const dlayer = require(".");
+const syncpipe = require("syncpipe");
+
+let fakeDriveTree = {
+	one: [ "/dev/1a", "/dev/1b" ],
+	two: [ "/dev/2a" ]
+};
+
+let invertedTree = {
+	"/dev/1a": "one",
+	"/dev/1b": "one",
+	"/dev/2a": "two"
+};
+
+let contextCounter = 1;
+
+// FIXME: Disallow type name conflicts!
+
+let moduleDrives = {
+	name: "Drives",
+	types: {
+		"sysquery.core.Drive": function ({ name }) {
+			return {
+				name: name
+			};
+		}
+	},
+	extensions: {
+		"sysquery.core.BlockDevice": {
+			drive: async function (_, { counter, $getProperty, $make }) {
+				console.log(`[context ${counter}] BlockDevice::drive`);
+
+				return $make("sysquery.core.Drive", {
+					name: invertedTree[await $getProperty(this, "path")]
+				});
+			}
+		}
+	},
+	root: {
+		hardware: {
+			drives: function ({ names }, { counter, $make }) {
+				console.log(`[context ${counter}] root::drives`);
+
+				return syncpipe(fakeDriveTree, [
+					_ => Object.entries(_),
+					_ => (names != null)
+						? _.filter(([ name, _devices ]) => names.includes(name))
+						: _,
+					_ => _.map(([ name, _devices ]) => $make("sysquery.core.Drive", { name }))
+				]);
+			}
+		}
+	},
+	makeContext: () => {
+		return {
+			counter: contextCounter++
+		};
+	}
+};
+
+let moduleBlockDevices = {
+	name: "Block Devices",
+	types: {
+		"sysquery.core.BlockDevice": function ({ path }) {
+			return {
+				path: path
+			};
+		}
+	},
+	extensions: {
+		"sysquery.core.Drive": {
+			blockDevices: async function (_, { counter, $getProperty, $make }) {
+				console.log(`[context ${counter}] Drive::blockDevices`);
+
+				return fakeDriveTree[await $getProperty(this, "name")].map((path) => {
+					return $make("sysquery.core.BlockDevice", { path });
+				});
+			}
+		}
+	},
+	root: {
+		hardware: {
+			blockDevices: function ({ paths }, { counter, $make }) {
+				console.log(`[context ${counter}] root::blockDevices`);
+
+				return syncpipe(fakeDriveTree, [
+					_ => Object.values(_),
+					_ => _.flat(),
+					_ => (paths != null)
+						? _.filter((path) => paths.includes(path))
+						: _,
+					_ => _.map((path) => $make("sysquery.core.BlockDevice", { path }))
+				]);
+			}
+		}
+	},
+	makeContext: () => {
+		return {
+			counter: contextCounter++
+		};
+	}
+};
+
+let api = dlayer({
+	schema: {
+		foo: {
+			bar: "baz"
+		}
+	},
+	modules: [ moduleBlockDevices, moduleDrives ]
+});
+
+return Promise.try(() => {
+	return api.query({
+		hardware: {
+			blockDevices: {
+				$arguments: { paths: [ "/dev/1b" ] },
+				path: true,
+				drive: {
+					name: true,
+					blockDevices: {
+						path: true
+					}
+				}
+			}
+		}
+	});
+}).then((result) => {
+	console.log("-- result:");
+	console.dir(result, {depth: null});
+}).catch((error) => {
+	console.log("Unhandled error:");
+	console.dir(error);
+});