From 87c95dc60a880d3ba42a5dd8c2b5e7cbc7151bf6 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Wed, 5 Jul 2023 01:14:06 +0200 Subject: [PATCH] dlayer: Add module system --- src/packages/dlayer/deep-merge.js | 38 ++++++++ src/packages/dlayer/docs.md | 2 + src/packages/dlayer/index.js | 59 ++++++++++-- src/packages/dlayer/load-modules.js | 128 ++++++++++++++++++++++++++ src/packages/dlayer/test-modules.js | 137 ++++++++++++++++++++++++++++ 5 files changed, 354 insertions(+), 10 deletions(-) create mode 100644 src/packages/dlayer/deep-merge.js create mode 100644 src/packages/dlayer/load-modules.js create mode 100644 src/packages/dlayer/test-modules.js diff --git a/src/packages/dlayer/deep-merge.js b/src/packages/dlayer/deep-merge.js new file mode 100644 index 0000000..4eb666f --- /dev/null +++ b/src/packages/dlayer/deep-merge.js @@ -0,0 +1,38 @@ +"use strict"; + +// A strict deep-merging implementation that *only* merges regular objects, and prevents prototype pollution + +function isObject(value) { + // TODO: Disallow special object types, for statically defined values (or just disallow specifying static values in the root schema in dlayer?) + return (value != null && typeof value === "object" && !Array.isArray(value)); +} + +module.exports = function deepMerge(a, b) { + let merged = Object.create(null); + let keys = new Set([ ... Object.keys(a), ... Object.keys(b) ]); + + for (let key of keys) { + // Technically over-blocks *any* 'constructor' key + if (key === "__proto__" || key === "constructor") { + continue; + } + + let valueA = a[key]; + let valueB = b[key]; + + if (isObject(valueA) && valueB === undefined) { + merged[key] = valueA; + } else if (isObject(valueB) && valueA === undefined) { + merged[key] = valueB; + } else if (isObject(valueA) && isObject(valueB)) { + merged[key] = deepMerge(valueA, valueB); + } else if (!isObject(valueA) && !isObject(valueB)) { + merged[key] = valueB ?? valueA; + } else { + // FIXME: Identifiable error type, and include the error path as well + throw new Error("Cannot merge non-object into object"); + } + } + + return merged; +}; \ No newline at end of file diff --git a/src/packages/dlayer/docs.md b/src/packages/dlayer/docs.md index a939ca6..86579b7 100644 --- a/src/packages/dlayer/docs.md +++ b/src/packages/dlayer/docs.md @@ -1,3 +1,5 @@ +A module system for your data sources. + ```js { system: { diff --git a/src/packages/dlayer/index.js b/src/packages/dlayer/index.js index 00341c4..fa3d032 100644 --- a/src/packages/dlayer/index.js +++ b/src/packages/dlayer/index.js @@ -6,16 +6,18 @@ const syncpipe = require("syncpipe"); const Result = require("../result"); const createCursor = require("./cursor"); +const deepMerge = require("./deep-merge"); +const loadModules = require("./load-modules"); // TODO: Bounded/unbounded recursion -// TODO: context +// TODO: Should we allow async context generation? Both in root schema *and* in modules // TODO: $required query predicate // TODO: Lazy query objects, which get initialized by calling a function that gets the parent object as an argument? This would not be serializable over the network! // FIXME: $getProperty, $getPropertyPath, maybe $resolveObject/$query? // FIXME: Allow setting an evaluation depth limit for queries, to limit eg. recursion // FIXME: recurseDepth, recurseLabel/recurseGoto -/* Data structure design: +/* Process design: - The process starts with: 1. A query tree, a nested object representing the query from the user @@ -97,10 +99,8 @@ function analyzeSubquery(subquery) { } function makeInstruction(cursor, queryKey) { - // let schemaKey = cursor.childQuery(queryKey)?.$key ?? queryKey; // $key is for handling aliases let childCursor = cursor.child(queryKey); let handler = childCursor.schema ?? cursor.schema.$anyKey; - // TODO: Maybe clean up all the .child stuff here by moving the `$key` logic into the cursor implementation instead, as it seems like nothing else in dlayer needs to care about the aliasing return { ... analyzeSubquery(childCursor.query), @@ -119,7 +119,7 @@ function assignErrorPath(error, cursor) { // TODO: build a sample todo-list schema for testing out fetches, mutations, and combinations of them, including on collections -function makeEnvironment(context) { +function makeEnvironment(context, getContextForModule) { function callHandler(instruction) { // NOTE: cursor is assumed to already be the key of the child let { schemaKey, handler, args, allowErrors, cursor } = instruction; @@ -127,7 +127,13 @@ function makeEnvironment(context) { if (handler != null) { return Promise.try(() => { // This calls the data provider in the schema - return Result.wrapAsync(() => maybeCall(handler, [ args, context ], cursor.parent.schema)); + if (handler.__moduleID != null) { + // Defined in a module + return Result.wrapAsync(() => maybeCall(handler.func, [ args, getContextForModule(handler.__moduleID) ], cursor.parent.schema)); + } else { + // Defined in the root schema + return Result.wrapAsync(() => maybeCall(handler, [ args, context ], cursor.parent.schema)); + } }).then((result) => { if (result.isOK) { return result.value(); @@ -199,7 +205,7 @@ function makeEnvironment(context) { let effectiveSubquery = (instruction.isRecursive) ? { ... cursor.query, ... subquery } : subquery; - + let finalValue = (instruction.isLeaf || value == null) ? value : applyToResultValue(instruction, value, effectiveSubquery); @@ -220,6 +226,9 @@ function makeEnvironment(context) { module.exports = function createDLayer(options) { // options = { schema, makeContext } + let loaded = loadModules(options.modules); + let schema = deepMerge(loaded.root, options.schema); + return { query: function (query, context) { let generatedContext = (options.makeContext != null) @@ -227,6 +236,7 @@ module.exports = function createDLayer(options) { : {}; function getProperty(object, property, args = {}) { + // TODO: Should this allow a single-argument, property-string-only variant for looking up properties on self? // FIXME: Validatem if (object == null) { throw new Error(`Empty object passed`); @@ -238,7 +248,30 @@ module.exports = function createDLayer(options) { // FIXME: Better error message with path throw new Error(`No key '${property}' exists in the schema`); } - } + } + + function make(typeID, args, existenceRequired) { + let type = loaded.types[typeID].func; + + if (type == null) { + if (existenceRequired === true) { + throw new Error(`No type named '${typeID}' exists`); + } else { + return; + } + } else { + let instance = type(args); + + if (loaded.extensions[typeID] != null) { + for (let [ key, extension ] of Object.entries(loaded.extensions[typeID])) { + // TODO: Possibly make this more performant by making it possible to do an Object.assign? Or does that not matter for performance? + instance[key] = extension.func; + } + } + + return instance; + } + } let combinedContext = { ... generatedContext, @@ -258,15 +291,21 @@ module.exports = function createDLayer(options) { return null; } }, object); + }, + $make: function (typeID, args) { + return make(typeID, args, true); + }, + $maybeMake: function (typeID, args) { + return make(typeID, args, false); } }; let cursor = createCursor({ query: query, - schema: options.schema + schema: schema }); - let evaluate = makeEnvironment(combinedContext); + let evaluate = makeEnvironment(combinedContext, loaded.makeContextFactory(combinedContext)); // FIXME: Currently, top-level errors do not get a path property assigned to them, because that assignment happens on nested calls above return evaluate(cursor); diff --git a/src/packages/dlayer/load-modules.js b/src/packages/dlayer/load-modules.js new file mode 100644 index 0000000..2700b4d --- /dev/null +++ b/src/packages/dlayer/load-modules.js @@ -0,0 +1,128 @@ +"use strict"; + +// const mergeByTemplate = require("merge-by-template"); +const deepMerge = require("./deep-merge"); +const syncpipe = require("syncpipe"); + +/* +Take a list of modules; each module specifies a name, schema root, types, type extensions, and a context factory function. Each module is internally assigned a unique ID. This unique ID is associated with each type factory and type extension method, and used as the key for a map of context factories; that way, upon invoking those methods, the module's own corresponding context can be injected. Only a single context should be created per module per request, so there should be a cache layer for the contexts (keyed by module ID), with each request creating a new cache. +*/ + +// NOTE: This can be global because we identify existing assignments by object identity, and that will never conflict +let numberedModules = new WeakMap(); +let currentModuleNumber = 0; + +function getModuleID(module) { + if (!numberedModules.has(module)) { + numberedModules.set(module, currentModuleNumber++); + } + + return numberedModules.get(module); +} + +function createTypeTracker() { + let typeFactories = {}; + + return { + add: function (module, name, factory) { + if (typeFactories[name] != null) { + let existingEntry = typeFactories[name]; + throw new Error(`Type '${name}' already exists (from module '${module.name}', already defined by module '${existingEntry.source.name}')`); + } else { + typeFactories[name] = { + source: module, + // No context provided to type factory functions for now, since they are not allowed to be async for now anyway + // FIXME: Maybe add a warning if the user returns a Promise from a factory, asking them to file a bug if they really need it? + // func: wrapModuleFunction(module, factory) + func: factory + }; + } + }, + get: function () { + return typeFactories; + } + }; +} + +function createExtensionTracker() { + let extendedTypes = {}; + + return { + add: function (module, type, name, method) { + if (extendedTypes[type] == null) { + extendedTypes[type] = {}; + } + + let extensions = extendedTypes[type]; + + if (extensions[name] != null) { + let existingEntry = extensions[name]; + throw new Error(`Type '${type}' already has a method extension named '${name}' (from module '${module.name}', already defined by module '${existingEntry.source.name}')`); + } else { + extensions[name] = { + source: module, + func: wrapModuleFunction(module, method) + }; + } + }, + get: function () { + return extendedTypes; + } + }; +} + +function wrapModuleFunction(module, func) { + return { __moduleID: getModuleID(module), func: func }; +} + +function defaultContext() { + // Fallback function that generates an empty context, for when a module doesn't specify a makeContext handler + return {}; +} + +module.exports = function (modules) { + // TODO: Eventually replace hand-crafted merging logic with merge-by-template, once it can support this usecase properly(tm) + // TODO: Fix merge-by-template so that reasonable error messages can be generated here, that are actually aware of eg. the conflicting key + + let types = createTypeTracker(); + let typeExtensions = createExtensionTracker(); + + let contextFactories = syncpipe(modules, [ + _ => _.map((module) => [ getModuleID(module), module.makeContext ?? defaultContext ]), + _ => new Map(_) + ]); + + let schemaRoots = modules.map((module) => module.root ?? {}); + + for (let module of modules) { + for (let [ type, factory ] of Object.entries(module.types)) { + types.add(module, type, factory); + } + + for (let [ type, extensions ] of Object.entries(module.extensions)) { + for (let [ name, method ] of Object.entries(extensions)) { + typeExtensions.add(module, type, name, method); + } + } + } + + return { + root: schemaRoots.reduce(deepMerge, {}), + types: types.get(), + extensions: typeExtensions.get(), + makeContextFactory: function (baseContext) { + let cache = new Map(); + + return function makeContextForModule(moduleID) { + if (!cache.has(moduleID)) { + cache.set(moduleID, { + ... baseContext, + ... contextFactories.get(moduleID)() + }); + } + + return cache.get(moduleID); + }; + } + }; +}; diff --git a/src/packages/dlayer/test-modules.js b/src/packages/dlayer/test-modules.js new file mode 100644 index 0000000..e25ea38 --- /dev/null +++ b/src/packages/dlayer/test-modules.js @@ -0,0 +1,137 @@ +"use strict"; + +const Promise = require("bluebird"); +const dlayer = require("."); +const syncpipe = require("syncpipe"); + +let fakeDriveTree = { + one: [ "/dev/1a", "/dev/1b" ], + two: [ "/dev/2a" ] +}; + +let invertedTree = { + "/dev/1a": "one", + "/dev/1b": "one", + "/dev/2a": "two" +}; + +let contextCounter = 1; + +// FIXME: Disallow type name conflicts! + +let moduleDrives = { + name: "Drives", + types: { + "sysquery.core.Drive": function ({ name }) { + return { + name: name + }; + } + }, + extensions: { + "sysquery.core.BlockDevice": { + drive: async function (_, { counter, $getProperty, $make }) { + console.log(`[context ${counter}] BlockDevice::drive`); + + return $make("sysquery.core.Drive", { + name: invertedTree[await $getProperty(this, "path")] + }); + } + } + }, + root: { + hardware: { + drives: function ({ names }, { counter, $make }) { + console.log(`[context ${counter}] root::drives`); + + return syncpipe(fakeDriveTree, [ + _ => Object.entries(_), + _ => (names != null) + ? _.filter(([ name, _devices ]) => names.includes(name)) + : _, + _ => _.map(([ name, _devices ]) => $make("sysquery.core.Drive", { name })) + ]); + } + } + }, + makeContext: () => { + return { + counter: contextCounter++ + }; + } +}; + +let moduleBlockDevices = { + name: "Block Devices", + types: { + "sysquery.core.BlockDevice": function ({ path }) { + return { + path: path + }; + } + }, + extensions: { + "sysquery.core.Drive": { + blockDevices: async function (_, { counter, $getProperty, $make }) { + console.log(`[context ${counter}] Drive::blockDevices`); + + return fakeDriveTree[await $getProperty(this, "name")].map((path) => { + return $make("sysquery.core.BlockDevice", { path }); + }); + } + } + }, + root: { + hardware: { + blockDevices: function ({ paths }, { counter, $make }) { + console.log(`[context ${counter}] root::blockDevices`); + + return syncpipe(fakeDriveTree, [ + _ => Object.values(_), + _ => _.flat(), + _ => (paths != null) + ? _.filter((path) => paths.includes(path)) + : _, + _ => _.map((path) => $make("sysquery.core.BlockDevice", { path })) + ]); + } + } + }, + makeContext: () => { + return { + counter: contextCounter++ + }; + } +}; + +let api = dlayer({ + schema: { + foo: { + bar: "baz" + } + }, + modules: [ moduleBlockDevices, moduleDrives ] +}); + +return Promise.try(() => { + return api.query({ + hardware: { + blockDevices: { + $arguments: { paths: [ "/dev/1b" ] }, + path: true, + drive: { + name: true, + blockDevices: { + path: true + } + } + } + } + }); +}).then((result) => { + console.log("-- result:"); + console.dir(result, {depth: null}); +}).catch((error) => { + console.log("Unhandled error:"); + console.dir(error); +});