WIP, fix string handling, support let attrsets, function argument defaults

run.js

@@ -10,4 +10,5 @@ assert(process.argv[2] != null);
 const nixFilePath = process.argv[2];
 const nixFile = fs.readFileSync(nixFilePath, "utf8");
 
-console.log(evaluate(nixFile));
+// console.log(evaluate(nixFile));
+evaluate(nixFile);

src/evaluate.js

@@ -20,6 +20,27 @@ module.exports = function evaluate(nixCode) {
 
 				return storedResult;
 			};
+		},
+		$$jsNix$handleArgument: function (name, _arg, defaultValue) {
+			// We need to evaluate the lazy wrapper for `arg`, as it is passed in as an attribute set; and since that is a value, it will have been wrapped.
+			const arg = _arg();
+
+			// FIXME: Improve this check, check for a (translated) attrset specifically
+			if (typeof arg === "object") {
+				// NOTE: We do *not* evaluate the actual attributes, nor the default value; them merely being present is enough for our case.
+				const value = arg[name];
+
+				if (value !== undefined) {
+					return value;
+				} else if (defaultValue !== undefined) {
+					return defaultValue;
+				} else {
+					throw new Error(`Missing required argument '${name}'`);
+				}
+			} else {
+				// FIXME: Improve error
+				throw new Error(`Must pass an attribute set to function that expects one`);
+			}
 		}
 	};
 

src/prepare-ast.js

@@ -3,6 +3,7 @@
 const matchValue = require("match-value");
 const asExpression = require("as-expression");
 
+// TODO: Refactor this
 function convertStringNodeParts(node) {
 	let fullText = node.text;
 	let currentIndex = node.startIndex;
@@ -32,7 +33,26 @@ function convertStringNodeParts(node) {
 		parts.push({ type: "NixInterpolationLiteral", value: value });
 	}
 
-	return parts;
+	return parts.map((part, i) => {
+		// FIXME: Verify that this logic is correct, and also holds up for actual interpolated strings
+		let isLiteralString = (part.type === "NixInterpolationLiteral");
+
+		if (isLiteralString) {
+			let strippedString = part.value;
+
+			if (i === 0 && isLiteralString) {
+				// Strip prefix quote
+				strippedString = strippedString.slice(1);
+			}
+
+			if (i === parts.length - 1 && isLiteralString) {
+				// Strip suffix quote
+				strippedString = strippedString.slice(0, -1);
+			}
+
+			return { ... part, value: strippedString };
+		}
+	});
 }
 
 function convertNode(node) {
@@ -118,6 +138,7 @@ function convertNode(node) {
 		attrset: "NixAttributeSet",
 		rec_attrset: "NixAttributeSet",
 		let: "NixLetIn",
+		let_attrset: "NixLetAttributeSet",
 		identifier: "NixIdentifier",
 		attr_identifier: "NixAttributeIdentifier",
 		attrpath: "NixAttributePath",

src/transformers/attribute-sets.js

@@ -5,25 +5,14 @@ const types = require("@babel/types");
 const template = require("@babel/template").default;
 const splitFilter = require("split-filter");
 
-const unpackExpression = require("./_unpack-expression");
+const unpackExpression = require("./util/unpack-expression");
 const NoChange = require("../astformer/actions/no-change");
 const printAst = require("../print-ast");
+const lazyWrapper = require("./templates/lazy-wrapper");
+const callLazyWrapper = require("./templates/call-lazy-wrapper");
 
 // TODO: Optimize lazy evaluation wrappers by only unpacking them selectively when used in an actual expression; in particular, that avoids the "wrapper that just calls another wrapper" overhead when passing attributes as function arguments
 
-// FIXME: Add expression parens!
-let tmplCallLazy = template(`
-	%%wrapper%%()
-`);
-
-let tmplLazyWrapper = template(`(
-	$$jsNix$memoize(() => %%expression%%)
-)`);
-
-let tmplLazyWrapperRecursive = template(`(
-	function() { return %%expression%%; }
-)`);
-
 let tmplExtend = template(`(
 	$$jsNix$extend(this ?? {}, %%object%%)
 )`);
@@ -41,17 +30,6 @@ let tmplRecursiveBinding = template(`
 	const %%name%% = %%expression%%;
 `);
 
-function lazyEvaluationWrapper(args) {
-	let { recursive, ... rest } = args;
-
-	// printAst(rest.expression);
-	let wrapper = (recursive)
-		? tmplLazyWrapperRecursive
-		: tmplLazyWrapper;
-
-	return unpackExpression(wrapper(rest));
-}
-
 
 function isDynamicBinding(binding) {
 	return binding.attrpath.attr[0].type !== "NixAttributeIdentifier";
@@ -72,9 +50,7 @@ module.exports = {
 				return node.attrpath.attr.reduce((last, identifier) => {
 					assert(identifier.type === "NixAttributeIdentifier");
 
-					return unpackExpression(tmplCallLazy({
+					return callLazyWrapper(types.memberExpression(last, types.identifier(identifier.name)));
-						wrapper: types.memberExpression(last, types.identifier(identifier.name))
-					}));
 				}, node.expression);
 			});
 		},
@@ -93,7 +69,7 @@ module.exports = {
 
 					return {
 						name: binding.attrpath.attr[0].name,
-						expression: lazyEvaluationWrapper({ recursive: isRecursive, expression: binding.expression })
+						expression: lazyWrapper(binding.expression)
 					};
 				});
 

src/transformers/desugar-attrsets.js

@@ -4,7 +4,7 @@
 const unreachable = require("@joepie91/unreachable")("jsNix");
 const NoChange = require("../astformer/actions/no-change");
 
-const { NixAttributeIdentifier, NixAttributeSet, NixBinding } = require("./_nix-types");
+const { NixAttributeIdentifier, NixAttributeSet, NixBinding } = require("./util/nix-types");
 
 function isAttributeSet(node) {
 	return (node.type === "NixAttributeSet");

src/transformers/desugar-inherits.js

@@ -4,7 +4,7 @@ const splitFilter = require("split-filter");
 const assert = require("assert");
 
 const NoChange = require("../astformer/actions/no-change");
-const nixTypes = require("./_nix-types");
+const nixTypes = require("./util/nix-types");
 
 module.exports = {
 	name: "desugar-inherits",
@@ -34,28 +34,44 @@ module.exports = {
 					);
 				});
 
-				let body = {
+				let inheritedAttributeBindings = inherits_.flatMap((inherit) => {
-					... node,
+					return inherit.names.map((name) => {
-					bind: [
+						return nixTypes.NixBinding(
-						... regularBindings,
+							[ nixTypes.NixAttributeIdentifier(name) ],
-						... inherits_.flatMap((inherit) => {
+							nixTypes.NixAttributeSelection(
-							return inherit.names.map((name) => {
+								nixTypes.NixIdentifier(inherit.tempName),
-								return nixTypes.NixBinding(
+								[ nixTypes.NixAttributeIdentifier(name) ]
-									[ nixTypes.NixAttributeIdentifier(name) ],
+							)
-									nixTypes.NixAttributeSelection(
+						);
-										nixTypes.NixIdentifier(inherit.tempName),
+					});
-										[ nixTypes.NixAttributeIdentifier(name) ]
+				});
-									)
+
-								);
+				if (node.recursive === false) {
-							});
+					// When not recursive, we need to ensure that the temporary variables are defined a scope *higher* than the attribute set that actually makes use of their properties, because we cannot access those variables from the attribute bindings otherwise.
-						})
+					
-					]
+					return nixTypes.NixLetIn(
-				};
+						letBindings,
+						{
+							... node,
+							bind: [
+								... regularBindings,
+								... inheritedAttributeBindings
+							]
+						}
+					);
+				} else {
+					// ... but when recursive, we *must* specify the temporary variables as part of the recursive attribute set itself, because otherwise we don't have the ability to bidirectionally reference between fixed attributes and inherited ones; see, for example, eval-okay-scope-7.nix.
+					
+					return {
+						... node,
+						bind: [
+							... regularBindings,
+							... letBindings,
+							... inheritedAttributeBindings
+						]
+					};
+				}
 
-				return nixTypes.NixLetIn(
-					letBindings,
-					body
-				);
 			}
 		}
 	}

src/transformers/desugar-let-attribute-set.js

@@ -0,0 +1,44 @@
+"use strict";
+
+const unreachable = require("@joepie91/unreachable");
+const assert = require("assert");
+const splitFilter = require("split-filter");
+const _nixTypes = require("./util/nix-types");
+
+function isValidBodyAttribute(binding) {
+	assert(binding.attrpath.type === "NixAttributePath");
+	assert(binding.attrpath.attr.length > 0);
+	assert(binding.attrpath.attr[0].type === "NixAttributeIdentifier");
+
+	if (binding.attrpath.attr[0].name === "body") {
+		if (binding.attrpath.attr.length > 1) {
+			throw unreachable("attribute paths should have been desugared");
+		} else {
+			return true;
+		}
+	} else {
+		return false;
+	}
+}
+
+module.exports = {
+	name: "desugar-let-attribute-set",
+	visitors: {
+		NixLetAttributeSet: (node) => {
+			// We save a bunch of complexity here by directly translating to a recursive attribute set instead of `let..in`; our JS representation of `let..in` is *functionally* identical to how a LetAttributeSet works. We just use a different attribute name to represent the returned evaluation.
+			
+			let [ bodyBindings, actualBindings ] = splitFilter(node.bind, (binding) => isValidBodyAttribute(binding));
+
+			if (bodyBindings.length === 0) {
+				// TODO: Display source code position + snippet
+				// FIXME: It's possible to specify the `body` with a dynamic key; this means that we can't actually do a compile-time check here, at least not reliably, and we need to insert a runtime guard if the compile-time check fails and dynamic attributes are specified
+				throw new Error(`Missing required 'body' attribute in LetAttributeSet`);
+			} else {
+				return _nixTypes.JSNixLet(
+					actualBindings,
+					bodyBindings[0].expression
+				);
+			}
+		}
+	}
+};

src/transformers/functions.js

@@ -4,22 +4,9 @@ const asExpression = require("as-expression");
 const assert = require("assert");
 const types = require("@babel/types");
 const template = require("@babel/template").default;
-const unpackExpression = require("./_unpack-expression");
+const unpackExpression = require("./util/unpack-expression");
-
+const lazyWrapper = require("./templates/lazy-wrapper");
-// NOTE: These are arrow functions because variable references within functions should always refer to the nearest scope; and since we use `this` to handle variable references within recursive attribute sets, we need to ensure that a function definition *does not* create its own `this` context.
+const callLazyWrapper = require("./templates/call-lazy-wrapper");
-let tmplFunctionDefinitionFormalsUniversal = template(`
-	((%%universal%%) => {
-		const %%formals%% = %%universal%%;
-		return %%body%%;
-	})
-`);
-
-// FIXME: Test that this template form actually works with our formals structure
-let tmplFunctionDefinitionFormals = template(`
-	((%%formals%%) => {
-		return %%body%%;
-	})
-`);
 
 let tmplFunctionDefinitionUniversal = template(`
 	((%%universal%%) => {
@@ -27,45 +14,48 @@ let tmplFunctionDefinitionUniversal = template(`
 	})
 `);
 
+let tmplFunctionDefinitionWithFormals = template(`(
+	(%%universal%%) => {
+		%%handleArguments%%;
+		
+		return %%body%%;
+	}
+)`);
+
+let tmplHandleArgument = template(`
+	const %%name%% = $$jsNix$handleArgument(%%nameString%%, %%universal%%, %%defaultArg%%);
+`);
+
 let tmplFunctionCall = template(`
 	(%%function%%(%%arg%%))
 `);
 
-// NOTE: Duplicated from `attribute-sets` for now
+function typesUndefined() {
-let tmplLazyWrapper = template(`(
+	// Apparently there's no undefinedLiteral???
-	$$jsNix$memoize(() => %%expression%%)
+	return types.unaryExpression("void", types.numericLiteral(0));
-)`);
+}
 
 function functionDefinition({ universal, formals, body }) {
-	let convertedFormals = asExpression(() => {
+	if (formals != null) {
-		if (formals != null) {
+		let defaultedUniversal = universal ?? "$$jsNix$tempArg";
-			return types.objectPattern(formals.map((formal) => {
-				return types.objectProperty(
-					types.identifier(formal),
-					types.identifier(formal)
-				);
-			}));
-		} else {
-			return undefined;
-		}
-	});
 
-	if (universal != null && formals != null) {
+		return tmplFunctionDefinitionWithFormals({
-		return tmplFunctionDefinitionFormalsUniversal({
+			universal: defaultedUniversal,
 			body: body,
-			universal: universal,
+			handleArguments: formals.map((formal) => {
-			formals: convertedFormals
+				return tmplHandleArgument({
+					universal: defaultedUniversal,
+					name: types.identifier(formal.name),
+					nameString: types.stringLiteral(formal.name),
+					defaultArg: formal.default ?? typesUndefined()
+				});
+			})
 		});
-	} else if (universal != null) {
+	} else {
 		return tmplFunctionDefinitionUniversal({
 			body: body,
 			universal: universal
-		});
+		});	
-	} else {
-		return tmplFunctionDefinitionFormals({
-			body: body,
-			formals: convertedFormals
-		});
 	}
 }
 
@@ -91,7 +81,13 @@ module.exports = {
 								return node.formals.formal.map((formal) => {
 									assert(formal.type === "NixUnpackedAttribute");
 									assert(formal.name.type === "Identifier");
-									return formal.name.name;
+									return {
+										name: formal.name.name,
+										// NOTE: This unwrap-and-rewrap dance is necessary to make out-of-order references work, like in eval-okay-scope-4.nix
+										default: (formal.default != null)
+											? lazyWrapper(callLazyWrapper(formal.default))
+											: undefined
+									};
 								});
 							}
 						}),
@@ -105,7 +101,7 @@ module.exports = {
 				return unpackExpression(
 					tmplFunctionCall({
 						function: node.function,
-						arg: unpackExpression(tmplLazyWrapper({ expression: node.argument }))
+						arg: lazyWrapper(node.argument)
 					})
 				);
 			});

src/transformers/index.js

@@ -10,14 +10,14 @@ const ConsumeNode = require("../astformer/actions/consume-node");
 const RemoveNode = require("../astformer/actions/remove-node");
 
 const asExpression = require("as-expression");
-const unpackExpression = require("./_unpack-expression");
+const unpackExpression = require("./util/unpack-expression");
 const printAst = require("../print-ast");
 
 
 
-// FIXME: Make strict mode! Otherwise objects will inherit from `global`
+// FIXME: Auto-generate argument list based on exposed API surface?
 let tmplModule = template(`
-	module.exports = function({ builtins, $$jsNix$memoize }) {
+	module.exports = function({ builtins, $$jsNix$memoize, $$jsNix$handleArgument }) {
 		return %%contents%%;
 	};
 `);
@@ -92,9 +92,10 @@ let trivial = {
 
 module.exports = [
 	require("./desugar-inherits"),
+	require("./desugar-attrsets"),
+	require("./desugar-let-attribute-set"),
 	require("./mangle-identifiers"),
 	require("./let-in"),
-	require("./desugar-attrsets"),
 	require("./literals"),
 	require("./functions"),
 	require("./attribute-sets"),

src/transformers/let-in.js

@@ -1,20 +1,14 @@
 "use strict";
 
-const nixTypes = require("./_nix-types");
+const nixTypes = require("./util/nix-types");
 
 module.exports = {
 	name: "let-in",
 	visitors: {
 		NixLetIn: (node) => {
-			return nixTypes.NixAttributeSelection(
+			return nixTypes.JSNixLet(
-				nixTypes.NixAttributeSet([
+				node.bind,
-					... node.bind,
+				node.body
-					nixTypes.NixBinding(
-						[ nixTypes.NixAttributeIdentifier("$$jsNix$letBody") ],
-						node.body
-					)
-				], true),
-				[ nixTypes.NixAttributeIdentifier("$$jsNix$letBody") ]
 			);
 		}
 	}

src/transformers/templates/call-lazy-wrapper.js

@@ -0,0 +1,12 @@
+"use strict";
+
+const template = require("@babel/template").default;
+const unpackExpression = require("../util/unpack-expression");
+
+let tmplCallLazy = template(`(
+	%%wrapper%%()
+)`);
+
+module.exports = function callLazyWrapper(wrapper) {
+	return unpackExpression(tmplCallLazy({ wrapper }));
+};

src/transformers/templates/lazy-wrapper.js

@@ -0,0 +1,15 @@
+"use strict";
+
+const template = require("@babel/template").default;
+
+const unpackExpression = require("../util/unpack-expression");
+
+let tmplLazyWrapper = template(`(
+	$$jsNix$memoize(() => %%expression%%)
+)`);
+
+module.exports = function lazyWrapper(expression) {
+	return unpackExpression(tmplLazyWrapper({
+		expression: expression
+	}));
+};

src/transformers/util/nix-types.js

@@ -63,5 +63,19 @@ let types = module.exports = {
 			bind: bindings,
 			body: body
 		};
+	},
+	// jsNix-specific structures
+	// FIXME: Should these be template modules instead?
+	JSNixLet: function (bindings, body) {
+		return types.NixAttributeSelection(
+			types.NixAttributeSet([
+				... bindings,
+				types.NixBinding(
+					[ types.NixAttributeIdentifier("$$jsNix$letBody") ],
+					body
+				)
+			], true),
+			[ types.NixAttributeIdentifier("$$jsNix$letBody") ]
+		);
 	}
 };

tests/upstream-nix.js

@@ -3,6 +3,7 @@
 const tape = require("tape-catch");
 const fs = require("fs");
 const path = require("path");
+const util = require("util");
 const evaluate = require("../src/evaluate");
 
 const NIX_SOURCE_REPO = process.env.NIX_SOURCE_REPO;
@@ -17,6 +18,14 @@ let tests = fs.readdirSync(testsPath)
 	.filter((entry) => entry.endsWith(".exp"))
 	.map((entry) => entry.replace(/\.exp$/, ""));
 
+function formatResultNode(node) {
+	if (typeof node === "string") {
+		return `"${node.replace(/"/g, '\\"')}"`;
+	} else {
+		return node.toString();
+	}
+}
+
 for (let test of tests) {
 	try {
 		let expression = fs.readFileSync(path.join(testsPath, `${test}.nix`), "utf8");
@@ -25,7 +34,7 @@ for (let test of tests) {
 		tape(`Nix upstream language tests - ${test}`, (test) => {
 			test.plan(1);
 
-			let result = evaluate(expression).value.toString();
+			let result = formatResultNode(evaluate(expression).value);
 
 			test.equals(expectedResult, result);
 		});