WIP

package.json

@@ -7,9 +7,11 @@
   "license": "MIT",
   "dependencies": {
     "as-expression": "^1.0.0",
+    "chalk": "^4",
     "es6-promise-try": "^1.0.2",
     "is-generator-function": "^1.0.10",
     "is-regex": "^1.1.4",
+    "map-obj": "^4",
     "match-value": "^1.1.0"
   },
   "devDependencies": {

poc-generators-2/core-ops/index.js

@@ -0,0 +1,37 @@
+"use strict";
+
+const mapObject = require("map-obj");
+const yieldcore = require("../yieldcore");
+
+let coreOps = {
+	literal: require("./literal"),
+	characterRange: require("./character-range"),
+	endOfInput: require("./end-of-input"),
+	either: require("./either"),
+	peek: require("./peek"),
+	test: require("./test"),
+	zeroOrMore: require("./zero-or-more"),
+	oneOrMore: require("./one-or-more"),
+	optional: require("./optional"),
+	until: require("./until"),
+	wholeMatch: require("./whole-match"),
+	// FIXME: Implement trackPosition
+};
+
+Object.assign(module.exports, {
+	... coreOps,
+	internalCall: mapObject(coreOps, (name, func) => {
+		return [
+			name,
+			function (properties, state, context) {
+				let instruction = {
+					__protocolKitInstruction: true,
+					type: name,
+					... properties,
+				};
+
+				return yieldcore.Internal(func(instruction, state, context), instruction);
+			}
+		];
+	})
+});

poc-generators-2/core-ops/one-or-more.js

@@ -1,11 +1,10 @@
 "use strict";
 
-const yieldcore = require("../yieldcore");
 const { NoMatch } = require("../symbols");
-const zeroOrMore = require("./zero-or-more");
+const coreOps = require("./index");
 
 module.exports = function* oneOrMore(instruction, state, context) {
-	let matches = yield yieldcore.Internal(zeroOrMore(instruction, state, context));
+	let matches = yield coreOps.internalCall.zeroOrMore({ rule: instruction.rule }, state, context);
 
 	// FIXME: NotEnoughInput propagation necessary here?
 	if (matches.length > 0) {

poc-generators-2/core-ops/test.js

@@ -1,11 +1,10 @@
 "use strict";
 
-const yieldcore = require("../yieldcore");
 const { NoMatch } = require("../symbols");
-const peek = require("./peek");
+const coreOps = require("./index");
 
 module.exports = function* test(instruction, state, context) {
-	let result = yield* yieldcore.Internal(peek(instruction, state, context)); // FIXME: Test that this actually works
+	let result = yield coreOps.internalCall.peek({ rule: instruction.rule }, state, context);
 
 	if (result === NoMatch) {
 		return false;

poc-generators-2/core-ops/zero-or-more.js

@@ -1,21 +1,27 @@
 "use strict";
 
-const { NotEnoughInput, NoMatch } = require("../symbols");
+const { NoMatch } = require("../symbols");
+const coreOps = require("./index");
+
+module.exports = function* zeroOrMore(instruction, state, context) {
+	let { rule } = instruction;
 
-module.exports = function* zeroOrMore(instruction, state) {
 	let matches = [];
 
 	while (true) {
-		let result = yield instruction.rule;
+		let reachedEnd = yield coreOps.internalCall.endOfInput({ rule: rule }, state, context);
+		// yieldcore.Internal(endOfInput(instruction, state, context), "endOfInput");
 
-		// FIXME: is NotEnoughInput handling actually necessary here? Wouldn't that be handled by the runtime hook?
-		if (result === NotEnoughInput) {
-			// Propagate, reparse later
-			return NotEnoughInput;
-		} else if (result === NoMatch) {
+		if (reachedEnd === true) {
 			break;
 		} else {
-			matches.push(result);
+			let result = yield instruction.rule;
+
+			if (result === NoMatch) {
+				break;
+			} else {
+				matches.push(result);
+			}
 		}
 	}
 

poc-generators-2/index.js

@@ -62,6 +62,7 @@ const matchValue = require("match-value");
 const util = require("util");
 const yieldcore = require("./yieldcore");
 const assert = require("assert");
+const chalk = require("chalk");
 
 const { NoMatch, NotEnoughInput } = require("./symbols");
 
@@ -80,7 +81,7 @@ function isInternalFrame(frame) {
 	if (process.env.DEBUG_PARSER_INTERNAL) {
 		return false;
 	} else {
-		return (frame.instruction === "internal");
+		return (frame.instruction === "internal" && typeof frame.name === "string" && frame.name.startsWith("_"));
 	}
 }
 
@@ -88,6 +89,15 @@ function getStackSize(stack) {
 	return stack.filter((frame) => !isInternalFrame(frame)).length;
 }
 
+function formatFrameInstruction(frame) {
+	if (frame.instruction === "internal") {
+		let formattedRule = util.inspect(frame.name, { colors: true, compact: true, breakLength: Infinity });
+		return chalk.blue(`[internal: ${formattedRule}]`);
+	} else {
+		return util.inspect(frame.instruction, { colors: true, compact: true, breakLength: Infinity });
+	}
+}
+
 module.exports = {
 	NoMatch: NoMatch,
 	parse: function parse(input, rootParser) {
@@ -121,22 +131,9 @@ module.exports = {
 	
 			let context = { startIndex: state.currentIndex };
 
-			let handler = matchValue.literal(rule.type, {
-				literal: require("./core-ops/literal"),
-				characterRange: require("./core-ops/character-range"),
-				endOfInput: require("./core-ops/end-of-input"),
-				either: require("./core-ops/either"),
-				peek: require("./core-ops/peek"),
-				test: require("./core-ops/test"),
-				zeroOrMore: require("./core-ops/zero-or-more"),
-				oneOrMore: require("./core-ops/one-or-more"),
-				optional: require("./core-ops/optional"),
-				until: require("./core-ops/until"),
-				wholeMatch: require("./core-ops/whole-match"),
-				// FIXME: Implement trackPosition
-			});
-
-			let result = yield* handler(rule, state, context);
+			let handler = matchValue.literal(rule.type, require("./core-ops"));
+
+			let result = yield yieldcore.Internal(handler(rule, state, context), rule);
 
 			// FIXME: Restore index when retrying a match after NotEnoughInput
 
@@ -171,7 +168,7 @@ module.exports = {
 				let frame = stack.at(-1);
 
 				if (process.env.DEBUG_PARSER && !isInternalFrame(frame)) {
-					console.log(`>> (${formatIndex()})` + "  ".repeat(getStackSize(stack)) + util.inspect(frame.instruction, { colors: true, compact: true, breakLength: Infinity }));
+					console.log(`>> (${formatIndex()})` + "  ".repeat(getStackSize(stack)) + formatFrameInstruction(frame));
 				}
 			}
 		});

poc-generators-2/test-grammar.js

@@ -1,7 +1,7 @@
 "use strict";
 
 const { parse } = require("./index");
-const { wholeMatch, either, peek } = require("./operations");
+const { wholeMatch, either, peek, oneOrMore } = require("./operations");
 
 function* A() {
 	return yield "hello";
@@ -12,7 +12,8 @@ function* Whitespace() {
 }
 
 function* B1() {
-	yield peek("world");
+	yield peek(oneOrMore("world"));
+	// yield peek("world");
 	return yield "world";
 }
 

poc-generators-2/yieldcore/index.js

@@ -34,10 +34,11 @@ module.exports = {
 			instruction: instruction
 		};
 	},
-	Internal: (generator) => {
+	Internal: (generator, name) => {
 		return {
 			__yieldcoreInternal: true,
-			generator: generator
+			generator: generator,
+			name: name
 		};
 		// FIXME: propagate NotEnoughInput, handle in custom handler as a retry
 	},
@@ -73,16 +74,18 @@ module.exports = {
 				instruction: instruction,
 				generator: instruction(),
 				done: false,
-				value: undefined
+				value: undefined,
+				name: undefined
 			});
 		}
 
-		function insertInternalInstruction(generator) {
+		function insertInternalInstruction(generator, name) {
 			increaseStack({
 				instruction: "internal",
 				generator: generator,
 				done: false,
-				value: undefined
+				value: undefined,
+				name: name // TODO: Rename this to `tag` or something instead? Doesn't have to be a name string
 			});
 		}
 
@@ -90,7 +93,6 @@ module.exports = {
 			while (finished === false && running === true) {
 				if (currentFrame.done === true) {
 					// This is a previously completed frame; it doesn't need any further processing
-					// console.log(`deferred-decreasing stack for ${currentFrame.instruction} with`, currentFrame.value);
 					decreaseStack(currentFrame.value);
 				} else {
 					// FIXME: Catch
@@ -103,13 +105,11 @@ module.exports = {
 						currentFrame.value = result.value;
 
 						let lastFrame = currentFrame;
-						// console.log(`decreasing stack for ${currentFrame.instruction} with`, result.value);
 						decreaseStack(result.value);
 
 						if (onReturn != null && lastFrame.instruction !== "internal") {
-							insertInternalInstruction(onReturn(result.value, lastFrame, stack));
+							insertInternalInstruction(onReturn(result.value, lastFrame, stack), "_onReturn");
 						} else {
-							// console.log(`setting action (no-onReturn) to ${result.value}`);
 							action = result.value;
 						}
 					} else if (isGenerator(result.value)) {
@@ -122,19 +122,17 @@ module.exports = {
 						insertInstruction(result.value);
 						continue; // Proceed with the next cycle immediately
 					} else if (isPromise(result.value)) {
-						// console.log(`setting action (promise) to ${result.value}`);
 						action = await result.value;
 					} else if (result.value === Pause) {
 						running = false;
 					} else if (result.value?.__yieldcoreInternal === true) {
-						insertInternalInstruction(result.value);
+						insertInternalInstruction(result.value.generator, result.value.name);
 						continue; // FIXME: Is this correct?
 					} else {
-						if (onYieldInstruction != null && currentFrame.instruction !== "internal") {
-							insertInternalInstruction(onYieldInstruction(result.value, currentFrame, stack));
+						if (onYieldInstruction != null) {
+							insertInternalInstruction(onYieldInstruction(result.value, currentFrame, stack), "_onYieldInstruction");
 						} else {
 							// TODO: Is there ever a good reason not to have an onYieldInstruction handler? Should we just disallow that case?
-							// console.log(`setting action (no-onYield) to ${result.value}`);
 							action = result.value;
 						}
 					

yarn.lock

@@ -130,7 +130,7 @@ callsites@^3.0.0:
   resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
   integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
 
-chalk@^4.0.0:
+chalk@^4, chalk@^4.0.0:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01"
   integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==
@@ -530,6 +530,11 @@ lodash.merge@^4.6.2:
   resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
   integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==
 
+map-obj@^4:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.3.0.tgz#9304f906e93faae70880da102a9f1df0ea8bb05a"
+  integrity sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==
+
 match-value@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/match-value/-/match-value-1.1.0.tgz#ad311ef8bbe2d344a53ec3104e28fe221984b98e"