From 0a74f6824ebca2836579de3c84c18523d93bd18d Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Mon, 7 Nov 2022 03:17:13 +0100 Subject: [PATCH] WIP --- package.json | 2 ++ poc-generators-2/core-ops/index.js | 37 +++++++++++++++++++++++ poc-generators-2/core-ops/one-or-more.js | 5 ++- poc-generators-2/core-ops/test.js | 5 ++- poc-generators-2/core-ops/zero-or-more.js | 24 +++++++++------ poc-generators-2/index.js | 33 +++++++++----------- poc-generators-2/test-grammar.js | 5 +-- poc-generators-2/yieldcore/index.js | 26 ++++++++-------- yarn.lock | 7 ++++- 9 files changed, 94 insertions(+), 50 deletions(-) create mode 100644 poc-generators-2/core-ops/index.js diff --git a/package.json b/package.json index 80430c6..214c678 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,11 @@ "license": "MIT", "dependencies": { "as-expression": "^1.0.0", + "chalk": "^4", "es6-promise-try": "^1.0.2", "is-generator-function": "^1.0.10", "is-regex": "^1.1.4", + "map-obj": "^4", "match-value": "^1.1.0" }, "devDependencies": { diff --git a/poc-generators-2/core-ops/index.js b/poc-generators-2/core-ops/index.js new file mode 100644 index 0000000..155e463 --- /dev/null +++ b/poc-generators-2/core-ops/index.js @@ -0,0 +1,37 @@ +"use strict"; + +const mapObject = require("map-obj"); +const yieldcore = require("../yieldcore"); + +let coreOps = { + literal: require("./literal"), + characterRange: require("./character-range"), + endOfInput: require("./end-of-input"), + either: require("./either"), + peek: require("./peek"), + test: require("./test"), + zeroOrMore: require("./zero-or-more"), + oneOrMore: require("./one-or-more"), + optional: require("./optional"), + until: require("./until"), + wholeMatch: require("./whole-match"), + // FIXME: Implement trackPosition +}; + +Object.assign(module.exports, { + ... coreOps, + internalCall: mapObject(coreOps, (name, func) => { + return [ + name, + function (properties, state, context) { + let instruction = { + __protocolKitInstruction: true, + type: name, + ... properties, + }; + + return yieldcore.Internal(func(instruction, state, context), instruction); + } + ]; + }) +}); diff --git a/poc-generators-2/core-ops/one-or-more.js b/poc-generators-2/core-ops/one-or-more.js index 28e0d34..77c7dea 100644 --- a/poc-generators-2/core-ops/one-or-more.js +++ b/poc-generators-2/core-ops/one-or-more.js @@ -1,11 +1,10 @@ "use strict"; -const yieldcore = require("../yieldcore"); const { NoMatch } = require("../symbols"); -const zeroOrMore = require("./zero-or-more"); +const coreOps = require("./index"); module.exports = function* oneOrMore(instruction, state, context) { - let matches = yield yieldcore.Internal(zeroOrMore(instruction, state, context)); + let matches = yield coreOps.internalCall.zeroOrMore({ rule: instruction.rule }, state, context); // FIXME: NotEnoughInput propagation necessary here? if (matches.length > 0) { diff --git a/poc-generators-2/core-ops/test.js b/poc-generators-2/core-ops/test.js index f4054de..2493b89 100644 --- a/poc-generators-2/core-ops/test.js +++ b/poc-generators-2/core-ops/test.js @@ -1,11 +1,10 @@ "use strict"; -const yieldcore = require("../yieldcore"); const { NoMatch } = require("../symbols"); -const peek = require("./peek"); +const coreOps = require("./index"); module.exports = function* test(instruction, state, context) { - let result = yield* yieldcore.Internal(peek(instruction, state, context)); // FIXME: Test that this actually works + let result = yield coreOps.internalCall.peek({ rule: instruction.rule }, state, context); if (result === NoMatch) { return false; diff --git a/poc-generators-2/core-ops/zero-or-more.js b/poc-generators-2/core-ops/zero-or-more.js index 41ad02f..8c6b0e5 100644 --- a/poc-generators-2/core-ops/zero-or-more.js +++ b/poc-generators-2/core-ops/zero-or-more.js @@ -1,21 +1,27 @@ "use strict"; -const { NotEnoughInput, NoMatch } = require("../symbols"); +const { NoMatch } = require("../symbols"); +const coreOps = require("./index"); + +module.exports = function* zeroOrMore(instruction, state, context) { + let { rule } = instruction; -module.exports = function* zeroOrMore(instruction, state) { let matches = []; while (true) { - let result = yield instruction.rule; + let reachedEnd = yield coreOps.internalCall.endOfInput({ rule: rule }, state, context); + // yieldcore.Internal(endOfInput(instruction, state, context), "endOfInput"); - // FIXME: is NotEnoughInput handling actually necessary here? Wouldn't that be handled by the runtime hook? - if (result === NotEnoughInput) { - // Propagate, reparse later - return NotEnoughInput; - } else if (result === NoMatch) { + if (reachedEnd === true) { break; } else { - matches.push(result); + let result = yield instruction.rule; + + if (result === NoMatch) { + break; + } else { + matches.push(result); + } } } diff --git a/poc-generators-2/index.js b/poc-generators-2/index.js index 01eb144..f7a96db 100644 --- a/poc-generators-2/index.js +++ b/poc-generators-2/index.js @@ -62,6 +62,7 @@ const matchValue = require("match-value"); const util = require("util"); const yieldcore = require("./yieldcore"); const assert = require("assert"); +const chalk = require("chalk"); const { NoMatch, NotEnoughInput } = require("./symbols"); @@ -80,7 +81,7 @@ function isInternalFrame(frame) { if (process.env.DEBUG_PARSER_INTERNAL) { return false; } else { - return (frame.instruction === "internal"); + return (frame.instruction === "internal" && typeof frame.name === "string" && frame.name.startsWith("_")); } } @@ -88,6 +89,15 @@ function getStackSize(stack) { return stack.filter((frame) => !isInternalFrame(frame)).length; } +function formatFrameInstruction(frame) { + if (frame.instruction === "internal") { + let formattedRule = util.inspect(frame.name, { colors: true, compact: true, breakLength: Infinity }); + return chalk.blue(`[internal: ${formattedRule}]`); + } else { + return util.inspect(frame.instruction, { colors: true, compact: true, breakLength: Infinity }); + } +} + module.exports = { NoMatch: NoMatch, parse: function parse(input, rootParser) { @@ -121,22 +131,9 @@ module.exports = { let context = { startIndex: state.currentIndex }; - let handler = matchValue.literal(rule.type, { - literal: require("./core-ops/literal"), - characterRange: require("./core-ops/character-range"), - endOfInput: require("./core-ops/end-of-input"), - either: require("./core-ops/either"), - peek: require("./core-ops/peek"), - test: require("./core-ops/test"), - zeroOrMore: require("./core-ops/zero-or-more"), - oneOrMore: require("./core-ops/one-or-more"), - optional: require("./core-ops/optional"), - until: require("./core-ops/until"), - wholeMatch: require("./core-ops/whole-match"), - // FIXME: Implement trackPosition - }); - - let result = yield* handler(rule, state, context); + let handler = matchValue.literal(rule.type, require("./core-ops")); + + let result = yield yieldcore.Internal(handler(rule, state, context), rule); // FIXME: Restore index when retrying a match after NotEnoughInput @@ -171,7 +168,7 @@ module.exports = { let frame = stack.at(-1); if (process.env.DEBUG_PARSER && !isInternalFrame(frame)) { - console.log(`>> (${formatIndex()})` + " ".repeat(getStackSize(stack)) + util.inspect(frame.instruction, { colors: true, compact: true, breakLength: Infinity })); + console.log(`>> (${formatIndex()})` + " ".repeat(getStackSize(stack)) + formatFrameInstruction(frame)); } } }); diff --git a/poc-generators-2/test-grammar.js b/poc-generators-2/test-grammar.js index 9070425..9920a61 100644 --- a/poc-generators-2/test-grammar.js +++ b/poc-generators-2/test-grammar.js @@ -1,7 +1,7 @@ "use strict"; const { parse } = require("./index"); -const { wholeMatch, either, peek } = require("./operations"); +const { wholeMatch, either, peek, oneOrMore } = require("./operations"); function* A() { return yield "hello"; @@ -12,7 +12,8 @@ function* Whitespace() { } function* B1() { - yield peek("world"); + yield peek(oneOrMore("world")); + // yield peek("world"); return yield "world"; } diff --git a/poc-generators-2/yieldcore/index.js b/poc-generators-2/yieldcore/index.js index dfbc682..3157006 100644 --- a/poc-generators-2/yieldcore/index.js +++ b/poc-generators-2/yieldcore/index.js @@ -34,10 +34,11 @@ module.exports = { instruction: instruction }; }, - Internal: (generator) => { + Internal: (generator, name) => { return { __yieldcoreInternal: true, - generator: generator + generator: generator, + name: name }; // FIXME: propagate NotEnoughInput, handle in custom handler as a retry }, @@ -73,16 +74,18 @@ module.exports = { instruction: instruction, generator: instruction(), done: false, - value: undefined + value: undefined, + name: undefined }); } - function insertInternalInstruction(generator) { + function insertInternalInstruction(generator, name) { increaseStack({ instruction: "internal", generator: generator, done: false, - value: undefined + value: undefined, + name: name // TODO: Rename this to `tag` or something instead? Doesn't have to be a name string }); } @@ -90,7 +93,6 @@ module.exports = { while (finished === false && running === true) { if (currentFrame.done === true) { // This is a previously completed frame; it doesn't need any further processing - // console.log(`deferred-decreasing stack for ${currentFrame.instruction} with`, currentFrame.value); decreaseStack(currentFrame.value); } else { // FIXME: Catch @@ -103,13 +105,11 @@ module.exports = { currentFrame.value = result.value; let lastFrame = currentFrame; - // console.log(`decreasing stack for ${currentFrame.instruction} with`, result.value); decreaseStack(result.value); if (onReturn != null && lastFrame.instruction !== "internal") { - insertInternalInstruction(onReturn(result.value, lastFrame, stack)); + insertInternalInstruction(onReturn(result.value, lastFrame, stack), "_onReturn"); } else { - // console.log(`setting action (no-onReturn) to ${result.value}`); action = result.value; } } else if (isGenerator(result.value)) { @@ -122,19 +122,17 @@ module.exports = { insertInstruction(result.value); continue; // Proceed with the next cycle immediately } else if (isPromise(result.value)) { - // console.log(`setting action (promise) to ${result.value}`); action = await result.value; } else if (result.value === Pause) { running = false; } else if (result.value?.__yieldcoreInternal === true) { - insertInternalInstruction(result.value); + insertInternalInstruction(result.value.generator, result.value.name); continue; // FIXME: Is this correct? } else { - if (onYieldInstruction != null && currentFrame.instruction !== "internal") { - insertInternalInstruction(onYieldInstruction(result.value, currentFrame, stack)); + if (onYieldInstruction != null) { + insertInternalInstruction(onYieldInstruction(result.value, currentFrame, stack), "_onYieldInstruction"); } else { // TODO: Is there ever a good reason not to have an onYieldInstruction handler? Should we just disallow that case? - // console.log(`setting action (no-onYield) to ${result.value}`); action = result.value; } diff --git a/yarn.lock b/yarn.lock index 63dedd2..cd6eff5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -130,7 +130,7 @@ callsites@^3.0.0: resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -chalk@^4.0.0: +chalk@^4, chalk@^4.0.0: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -530,6 +530,11 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== +map-obj@^4: + version "4.3.0" + resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.3.0.tgz#9304f906e93faae70880da102a9f1df0ea8bb05a" + integrity sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ== + match-value@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/match-value/-/match-value-1.1.0.tgz#ad311ef8bbe2d344a53ec3104e28fe221984b98e"