Compare commits

...

2 Commits

@ -4,7 +4,7 @@
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
"test": "tape tests/**/* | tap-difflet"
},
"repository": {
"type": "git",
@ -32,11 +32,14 @@
"map-obj": "^5.0.0",
"match-value": "^1.1.0",
"split-filter": "^1.1.3",
"tap-difflet": "^0.7.2",
"tape": "^5.5.0",
"tree-sitter-javascript": "^0.19.0",
"tree-sitter-nix": "cstrahan/tree-sitter-nix"
},
"devDependencies": {
"@joepie91/eslint-config": "^1.1.0",
"eslint": "^8.8.0"
"eslint": "^8.8.0",
"tape-catch": "^1.0.6"
}
}

@ -3,44 +3,11 @@
const fs = require("fs");
const assert = require("assert");
const transpile = require("./src/transpile");
const measureTime = require("./src/astformer/util/measure-time");
const evaluate = require("./src/evaluate");
assert(process.argv[2] != null);
const nixFilePath = process.argv[2];
const nixFile = fs.readFileSync(nixFilePath, "utf8");
let transpiled = transpile(nixFile);
const api = {
builtins: {},
$$jsNix$memoize: function (func) {
let isCalled = false;
let storedResult;
return function (arg) {
if (isCalled === false) {
storedResult = func(arg);
isCalled = true;
}
return storedResult;
};
}
};
// TODO: Switch to Node `vm` API instead, and check whether there's a polyfill for it for non-Node environments, build a custom one if not
const context = { module: {}, exports: {} };
context.module.exports = exports;
new Function("module", transpiled)(context.module);
console.log("-- EVALUATION RESULT:");
// Warm-up for hot VM performance testing
// for (let i = 0; i < 10000; i++) {
// context.module.exports(api);
// }
console.log(measureTime(() => context.module.exports(api)));
console.log(evaluate(nixFile));

@ -196,7 +196,7 @@ module.exports = function optimizeTree(ast, optimizers) {
debuggers[debuggerName](`Node of type '${typeOf(node)}' replaced by node of type '${typeOf(result)}'`);
if (iterations >= EVALUATION_LIMIT) {
throw new Error(`Exceeded evaluation limit in optimizer ${debuggerName}; aborting optimization. If you are a user of raqb, please report this as a bug. If you are writing an optimizer, make sure that your optimizer eventually stabilizes on a terminal condition (ie. NoChange)!`);
throw new Error(`Exceeded evaluation limit in optimizer ${debuggerName}; aborting optimization. If you are a user of this software, please report this as a bug. If you are a developer writing an optimizer, make sure that your optimizer eventually stabilizes on a terminal condition (ie. NoChange)!`);
} else {
return handleASTNode(result, iterations + 1, path, initialStateLog, context);
}

@ -0,0 +1,45 @@
"use strict";
const measureTime = require("./astformer/util/measure-time");
const transpile = require("./transpile");
module.exports = function evaluate(nixCode) {
let transpiled = transpile(nixCode);
const api = {
builtins: {},
$$jsNix$memoize: function (func) {
let isCalled = false;
let storedResult;
return function (arg) {
if (isCalled === false) {
storedResult = func(arg);
isCalled = true;
}
return storedResult;
};
}
};
// TODO: Switch to Node `vm` API instead, and check whether there's a polyfill for it for non-Node environments, build a custom one if not
const context = { module: {}, exports: {} };
context.module.exports = exports;
new Function("module", transpiled)(context.module);
// Warm-up for hot VM performance testing
// for (let i = 0; i < 10000; i++) {
// context.module.exports(api);
// }
let result = measureTime(() => context.module.exports(api));
if (process.env.DEBUG_NIX) {
console.log("-- EVALUATION RESULT:");
console.log(result);
}
return result;
};

@ -0,0 +1,42 @@
"use strict";
const assert = require("assert");
const acceptableFirstCharacters = /^[a-zA-Z_]$/;
const acceptableSubsequentCharacters = /^[0-9a-zA-Z_]$/;
// This function deterministically and statelessly translates a name such that it is guaranteed to be valid in all identifier positions in JS.
// TODO: This can probably be made more performant...
function mangleCharacter(character) {
return (character === "$")
? "$$"
: `$${character.codePointAt(0)}`
}
module.exports = function mangleName(name) {
assert(name.length > 0);
// FIXME: Tag an identifier of this type with an internal property instead
if (name.startsWith("$$jsNix$")) {
return name;
} else {
let completedFirstCharacter = false;
return Array.from(name)
.map((character) => {
if (!completedFirstCharacter) {
completedFirstCharacter = true;
return (acceptableFirstCharacters.test(character))
? character
: mangleCharacter(character);
} else {
return (acceptableSubsequentCharacters.test(character))
? character
: mangleCharacter(character);
}
})
.join("");
}
};

@ -95,6 +95,15 @@ function convertNode(node) {
result.recursive = false;
}
if (type === "inherit") {
// We're inheriting from scope here, so the source expression is explicitly not set
result.expression = null;
} else if (type === "inherit_from") {
// Already set
}
// console.log(result);
result.type = matchValue(result.type, {
source_expression: "NixProgram",
token: "NixAnonymousToken",
@ -112,6 +121,10 @@ function convertNode(node) {
identifier: "NixIdentifier",
attr_identifier: "NixAttributeIdentifier",
attrpath: "NixAttributePath",
inherit: "NixInherit",
inherit_from: "NixInherit",
attrs_inherited: "NixInheritAttributes",
attrs_inherited_from: "NixInheritAttributes",
bind: "NixBinding",
binary: "NixBinaryOperation",
app: "NixFunctionCall",

@ -28,13 +28,22 @@ let types = module.exports = {
};
},
NixAttributePath: function (attributes) {
// NOTE: This does not accept an array of strings! Instead, specify as an array of NixAttributeIdentifiers
// TODO: Add proper validation for this
assert(Array.isArray(attributes));
assert(!attributes.some((attribute) => typeof attribute === "string"));
return {
type: "NixAttributePath",
attr: attributes
};
},
NixIdentifier: function (name) {
return {
type: "NixIdentifier",
name: name
};
},
NixAttributeIdentifier: function (name) {
return {
type: "NixAttributeIdentifier",
@ -47,5 +56,12 @@ let types = module.exports = {
attrpath: types.NixAttributePath(attributePath),
expression: expression
};
},
NixLetIn: function (bindings, body) {
return {
type: "NixLetIn",
bind: bindings,
body: body
};
}
};

@ -3,10 +3,14 @@
const assert = require("assert");
const types = require("@babel/types");
const template = require("@babel/template").default;
const splitFilter = require("split-filter");
const unpackExpression = require("./_unpack-expression");
const NoChange = require("../astformer/actions/no-change");
const printAst = require("../print-ast");
// TODO: Optimize lazy evaluation wrappers by only unpacking them selectively when used in an actual expression; in particular, that avoids the "wrapper that just calls another wrapper" overhead when passing attributes as function arguments
// FIXME: Add expression parens!
let tmplCallLazy = template(`
%%wrapper%%()
@ -81,9 +85,10 @@ module.exports = {
return defer((node) => {
let isRecursive = node.recursive;
let hasDynamicBindings = node.bind.some((binding) => isDynamicBinding(binding));
let bindings = node.bind.map((binding) => {
let [ dynamicNodes, staticNodes ] = splitFilter(node.bind, (binding) => isDynamicBinding(binding));
let staticBindings = staticNodes.map((binding) => {
assert(binding.attrpath.attr.length === 1); // Nested attributes should have been desugared by this point
return {
@ -92,17 +97,18 @@ module.exports = {
};
});
if (hasDynamicBindings) {
if (dynamicNodes.length > 0) {
printAst(node);
throw new Error(`UNIMPLEMENTED: Dynamic bindings are not supported yet`);
} else if (isRecursive) {
return unpackExpression(tmplScopeWrapper({
bindings: bindings.map(({ name, expression }) => {
bindings: staticBindings.map(({ name, expression }) => {
return tmplRecursiveBinding({
name: name,
expression: expression
});
}),
object: types.objectExpression(bindings.map(({ name }) => {
object: types.objectExpression(staticBindings.map(({ name }) => {
return types.objectProperty(
types.stringLiteral(name),
types.identifier(name)
@ -110,7 +116,7 @@ module.exports = {
}))
}));
} else {
let object = types.objectExpression(bindings.map(({ name, expression }) => {
let object = types.objectExpression(staticBindings.map(({ name, expression }) => {
return types.objectProperty(
types.stringLiteral(name),
// TODO: Remove the isRecursive distinction here once heap-of-vars works

@ -1,6 +1,7 @@
"use strict";
const unreachable = require("@joepie91/unreachable")("jsNix");
const NoChange = require("../astformer/actions/no-change");
const { NixAttributeIdentifier, NixAttributeSet, NixBinding } = require("./_nix-types");
@ -44,39 +45,42 @@ module.exports = {
let dynamicBindings = [];
for (let binding of node.bind) {
if (binding.attrpath.attr.length > 1) {
neededDesugaring = true;
}
let firstAttribute = binding.attrpath.attr[0];
let isStaticAttribute = firstAttribute.type === "NixAttributeIdentifier";
if (isStaticAttribute) {
let attributeName = firstAttribute.name;
let existingBinding = newStaticBindings[attributeName];
if (existingBinding == null) {
newStaticBindings[attributeName] = NixBinding(
[ firstAttribute ],
unpackBindingValue(binding)
);
} else {
if (isAttributeSet(existingBinding.expression) && isAttributeSet(binding.expression)) {
neededDesugaring = true;
newStaticBindings[attributeName] = mergeBindings(attributeName, existingBinding, binding);
if (binding.type === "NixBinding") {
if (binding.attrpath.attr.length > 1) {
neededDesugaring = true;
}
let firstAttribute = binding.attrpath.attr[0];
let isStaticAttribute = firstAttribute.type === "NixAttributeIdentifier";
if (isStaticAttribute) {
let attributeName = firstAttribute.name;
let existingBinding = newStaticBindings[attributeName];
if (existingBinding == null) {
newStaticBindings[attributeName] = NixBinding(
[ firstAttribute ],
unpackBindingValue(binding)
);
} else {
throw new Error(`Key '${attributeName}' was specified twice, but this is only allowed when both values are an attribute set`);
if (isAttributeSet(existingBinding.expression) && isAttributeSet(binding.expression)) {
neededDesugaring = true;
newStaticBindings[attributeName] = mergeBindings(attributeName, existingBinding, binding);
} else {
throw new Error(`Key '${attributeName}' was specified twice, but this is only allowed when both values are an attribute set`);
}
}
} else {
// FIXME: Needs runtime check, need to *always* construct objects at runtime when dynamic bindings are involved
dynamicBindings.push(binding);
}
} else {
// FIXME: Needs runtime check, need to *always* construct objects at runtime when dynamic bindings are involved
dynamicBindings.push(binding);
unreachable(`unrecognized binding type: ${binding.type}`);
}
}
let staticBindingList = Object.entries(newStaticBindings).map(([ key, value ]) => {
// console.log([ key, value ]);
return value;
});

@ -0,0 +1,62 @@
"use strict";
const splitFilter = require("split-filter");
const assert = require("assert");
const NoChange = require("../astformer/actions/no-change");
const nixTypes = require("./_nix-types");
module.exports = {
name: "desugar-inherits",
visitors: {
NixAttributeSet: (node) => {
let [ inherits, regularBindings ] = splitFilter(node.bind, (binding) => binding.type === "NixInherit");
if (inherits.length === 0) {
return NoChange;
} else {
let tempCounter = 0;
let inherits_ = inherits.map((inherit) => {
return {
tempName: `$$jsNix$temp$${tempCounter++}`,
sourceExpression: inherit.expression,
names: inherit.attrs.attr.map((attribute) => {
assert(attribute.type === "NixAttributeIdentifier");
return attribute.name;
})
};
});
let letBindings = inherits_.map((inherit) => {
return nixTypes.NixBinding(
[ nixTypes.NixAttributeIdentifier(inherit.tempName) ],
inherit.sourceExpression
);
});
let body = {
... node,
bind: [
... regularBindings,
... inherits_.flatMap((inherit) => {
return inherit.names.map((name) => {
return nixTypes.NixBinding(
[ nixTypes.NixAttributeIdentifier(name) ],
nixTypes.NixAttributeSelection(
nixTypes.NixIdentifier(inherit.tempName),
[ nixTypes.NixAttributeIdentifier(name) ]
)
);
});
})
]
};
return nixTypes.NixLetIn(
letBindings,
body
);
}
}
}
};

@ -6,7 +6,6 @@ const types = require("@babel/types");
const template = require("@babel/template").default;
const unpackExpression = require("./_unpack-expression");
// TODO: Memoize every function
// NOTE: These are arrow functions because variable references within functions should always refer to the nearest scope; and since we use `this` to handle variable references within recursive attribute sets, we need to ensure that a function definition *does not* create its own `this` context.
let tmplFunctionDefinitionFormalsUniversal = template(`
((%%universal%%) => {

@ -91,6 +91,8 @@ let trivial = {
};
module.exports = [
require("./desugar-inherits"),
require("./mangle-identifiers"),
require("./let-in"),
require("./desugar-attrsets"),
require("./literals"),

@ -7,5 +7,13 @@ module.exports = {
visitors: {
NixIntegerLiteral: (node) => types.numericLiteral(parseInt(node.value)),
NixFloatLiteral: (node) => types.numericLiteral(parseFloat(node.value)),
NixStringLiteral: (node) => {
if (node.parts.length === 1 && node.parts[0].type === "NixInterpolationLiteral") {
// Fast case; just a simple, non-interpolated string literal
return types.stringLiteral(node.parts[0].value);
} else {
throw new Error(`Unimplemented: string interpolation is not supported yet`);
}
}
}
};

@ -0,0 +1,24 @@
"use strict";
const NoChange = require("../astformer/actions/no-change");
const mangleName = require("../mangle-name");
function mangleNode(node) {
if (node._isMangled) {
return NoChange;
} else {
return {
... node,
name: mangleName(node.name),
_isMangled: true
};
}
}
module.exports = {
name: "mangle-identifiers",
visitors: {
NixAttributeIdentifier: mangleNode,
NixIdentifier: mangleNode
}
};

@ -0,0 +1,40 @@
"use strict";
const tape = require("tape-catch");
const fs = require("fs");
const path = require("path");
const evaluate = require("../src/evaluate");
const NIX_SOURCE_REPO = process.env.NIX_SOURCE_REPO;
if (NIX_SOURCE_REPO == null) {
throw new Error(`To run the upstream Nix language tests, you must specify a NIX_SOURCE_REPO environment variable, that points at the root of a local checkout of the Git repository for Nix`);
}
const testsPath = path.join(NIX_SOURCE_REPO, "tests/lang");
let tests = fs.readdirSync(testsPath)
.filter((entry) => entry.endsWith(".exp"))
.map((entry) => entry.replace(/\.exp$/, ""));
for (let test of tests) {
try {
let expression = fs.readFileSync(path.join(testsPath, `${test}.nix`), "utf8");
let expectedResult = fs.readFileSync(path.join(testsPath, `${test}.exp`), "utf8").replace(/\n$/, "");
tape(`Nix upstream language tests - ${test}`, (test) => {
test.plan(1);
let result = evaluate(expression).value.toString();
test.equals(expectedResult, result);
});
} catch (error) {
// FIXME: This would currently cause ENOENTs during evaluation (eg. reading a file from Nix itself) to be ignored
if (error.code === "ENOENT") {
// skip
} else {
throw error;
}
}
}

File diff suppressed because it is too large Load Diff
Loading…
Cancel
Save