Browse Source

Initial commit

master
Sven Slootweg 5 months ago
commit
32a13e668a
  1. 3
      .eslintrc
  2. 2
      .gitignore
  3. 183
      mappings.txt
  4. 8
      notes.txt
  5. 42
      package.json
  6. 4
      roadmap.txt
  7. 24
      run.js
  8. 1
      samples/attrset-binary.nix
  9. 1
      samples/attrset-nested-rec-parent.nix
  10. 1
      samples/attrset-nested-rec.nix
  11. 1
      samples/attrset-nested.nix
  12. 1
      samples/attrset-self-reference.nix
  13. 6
      samples/attrsets.nix
  14. 4
      samples/complex-rec-scope.nix
  15. 1
      samples/sample-function.nix
  16. 6
      src/astformer/actions/consume-node.js
  17. 3
      src/astformer/actions/no-change.js
  18. 3
      src/astformer/actions/remove-node.js
  19. 20
      src/astformer/combine-optimizers.js
  20. 16
      src/astformer/create-debuggers.js
  21. 27
      src/astformer/handler-tracker.js
  22. 361
      src/astformer/index.js
  23. 12
      src/astformer/timings-tracker.js
  24. 11
      src/astformer/util/concat.js
  25. 19
      src/astformer/util/measure-time.js
  26. 5
      src/astformer/util/merge.js
  27. 12
      src/astformer/util/type-of.js
  28. 14
      src/parse.js
  29. 131
      src/prepare-ast.js
  30. 39
      src/print-ast.js
  31. 44
      src/transformers/_nix-types.js
  32. 9
      src/transformers/_unpack-expression.js
  33. 100
      src/transformers/attribute-sets.js
  34. 91
      src/transformers/desugar-attrsets.js
  35. 110
      src/transformers/functions.js
  36. 108
      src/transformers/index.js
  37. 11
      src/transformers/literals.js
  38. 45
      src/transpile.js
  39. 12
      testers/parse.js
  40. 14
      testers/transform.js
  41. 19
      todo-wrong.txt
  42. 1700
      yarn.lock

3
.eslintrc

@ -0,0 +1,3 @@
{
"extends": "@joepie91/eslint-config"
}

2
.gitignore

@ -0,0 +1,2 @@
node_modules
private-notes.txt

183
mappings.txt

@ -0,0 +1,183 @@
Global notes:
- When a JS construct has a statement form and expression form, the transpiler should *always* produce the expression form, to match Nix semantics. This can be forced in @babel/tmpl by wrapping the template in parentheses; these parentheses will not be present in the final output.
- "Expression wrapping" in this document refers to the practice of taking a syntax construct in JS that only exists in statement form (ie. no return value), and wrapping it in an IIFE using an internal `return` to translate that to expression form. This is necessary to match the semantics of Nix, where everything is an expression.
- "Readability" refers not only to how easy it is to literally read the transpiler output, but also to the clarity of eg. stacktraces when an error occurs. *Ideally*, it should be possible for a user to debug their Nix code from a JS stacktrace without needing any sourcemaps and without needing to look at the transpiler output.
- Identifiers which have special semantic meaning in JS but not in Nix, need to be prefixed such that Nix code cannot accidentally try to access them; for example, `this` and `function`.
- Runtime guards may be needed to prevent runtime type mismatches; Nix appears to be stricter here than JS.
- Call a non-function: error in both
- Specify undeclared named parameters to a function: error in Nix (without a rest parameter), allowed in JS
- Add a number to a string:
================
# `with` statement
Nix
with pkgs; <expression>
JS
(() => {
with (pkgs) {
return <expression>;
}
})()
Notes:
- Unlike in Nix, `with` is a statement in JS, not an expression, therefore expression wrapping is used.
- Strict mode in JS does not allow the use of `with`, so strict mode cannot be used. If it were ever necessary to do so, an alternative approach is to emulate the behaviour of `with` by translating all contained variable lookups to a `pkgs.foo ?? foo` format, but this would significantly reduce output readability and increase translation complexity.
================
# Attribute set literals
Nix
{
inherit foo;
inherit (other) bar;
a = 1;
b = a;
}
JS
{
foo: () => foo,
bar: () => other.bar,
a: () => 1,
b: () => this.?a ?? a
}
Notes:
- Lazy evaluation semantics require wrapping the values in a function, even though they are static values.
- `this.a` is not used to this object, but rather to any potential higher-level recursive attribute sets (see below)
================
# *Recursive* attribute set literals
Nix
rec { a = 1; b = a; }
JS
{
a: function() { return 1; },
b: function() { return this.a ?? a; }
}
Notes:
- Using regular functions instead of arrow functions; since a regular attribute set can refer to its own properties, a new `this` context needs to be created that refers to the object on which the lazy-wrapper is called.
-
================
# Attribute set merging
Nix
a // b // c
JS
{ ... a, ... b, ... c }
================
# List merging
Nix
a ++ b ++ c
JS
[ ... a, ... b, ... c ]
================
# Function definition
Nix
foo: <body>
JS
function(foo) { return <body>; }
Notes:
- FIXME: Is using a regular function (with its own `this` context) actually correct here? Or should it be an arrow function instead?
================
# Function definition with set pattern (named parameters)
Nix
{ foo, bar }: <body>
JS
function({ foo, bar }) { return <body>; }
Notes:
- The rest parameter (ellipsis) does not have or need an equivalent JS representation.
================
# Function definition with set pattern *and* a single identifier (ie. all named parameters as an attribute set)
Nix
{ foo, bar } @ all: <body>
JS
function(all) {
const { foo, bar } = all;
return <body>;
}
================
# Conditionals
Nix
if a then b else c
JS
(a) ? b : c
Notes:
- A ternary is used here despite the worse readability (compared to an if statement), because a ternary is natively an expression but an if statement is not. Therefore, this avoids an extra level of expression wrapping.
================
# `let ... in`
Nix
let a = 1; b = 2; in <expression>
JS
(() => {
let a = () => 1;
let b = () => 2;
return <expression>;
})()
Notes:
- Expression wrapping is used here to create a new scope for the evaluation of this expression; in Nix, a `let` binding only applies for the expression to which it is prefixed, not to the function scope like in JS, nor does JS have an equivalent native block scope for variable bindings.
- Bindings are *also* lazily evaluated
================
# Path literals
Nix
/some/path
JS
$core.evaluatePath("/some/path")
Notes:
- Returns a computed store path immediately as a string, and internally queues up an evaluation/build
================
# List literals
Nix
[ 1 2 3 (4 + 5) ]
JS
[ 1, 2, 3, 4 + 5 ]

8
notes.txt

@ -0,0 +1,8 @@
need to use regular functions for `rec {}` getters but arrow functions for `{}` getters, to ensure that using `this` for property access works correctly
ADD TO TESTS:
let a = 0; in rec { a = 1; foo = bar: a * 2 }
# 2
let a = 0; in rec { a = 1; foo = let func = bar: a * 2; in rec { a = 3; baz = func; }; }
# 2

42
package.json

@ -0,0 +1,42 @@
{
"name": "nix-in-node",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "git@git.cryto.net:joepie91/nix-in-node.git"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@babel/core": "^7.15.5",
"@babel/generator": "^7.15.4",
"@babel/template": "^7.15.4",
"@babel/traverse": "^7.15.4",
"@babel/types": "^7.15.6",
"@joepie91/unreachable": "^1.0.0",
"as-expression": "^1.0.0",
"assure-array": "^1.0.0",
"chalk": "^4.1.2",
"concat-arrays": "^2.0.0",
"default-value": "^1.0.0",
"estree-walker": "^2",
"find-last": "^1.0.0",
"fix-esm": "^1.0.1",
"is-plain-obj": "^4.0.0",
"map-obj": "^5.0.0",
"match-value": "^1.1.0",
"split-filter": "^1.1.3",
"tree-sitter-javascript": "^0.19.0",
"tree-sitter-nix": "cstrahan/tree-sitter-nix"
},
"devDependencies": {
"@joepie91/eslint-config": "^1.1.0",
"eslint": "^8.8.0"
}
}

4
roadmap.txt

@ -0,0 +1,4 @@
- convert rec attrset to use heap-of-vars
- wrap function args in lazy wrapper + unwrap identifier access?
- memoize functions
- let..in -> rec attrset

24
run.js

@ -0,0 +1,24 @@
"use strict";
const fs = require("fs");
const assert = require("assert");
const transpile = require("./src/transpile");
assert(process.argv[2] != null);
const nixFilePath = process.argv[2];
const nixFile = fs.readFileSync(nixFilePath, "utf8");
let transpiled = transpile(nixFile);
console.log("-- EVALUATION RESULT:");
console.log(eval(transpiled)({
builtins: {},
$$jsNix$extend: function (base, props) {
let newObject = Object.create(base);
Object.assign(newObject, props);
return newObject;
}
}));

1
samples/attrset-binary.nix

@ -0,0 +1 @@
rec { x = { a = 1; b = x.a + 1; c = x.b + 1; }; }.x

1
samples/attrset-nested-rec-parent.nix

@ -0,0 +1 @@
rec { a = x: x + c; c = 2; b = rec { c = 3; d = a c; }; }.b.d

1
samples/attrset-nested-rec.nix

@ -0,0 +1 @@
rec { a = 1; b = rec { c = a; }; }.b.c

1
samples/attrset-nested.nix

@ -0,0 +1 @@
rec { a = 1; b = { c = a; }; }.b.c

1
samples/attrset-self-reference.nix

@ -0,0 +1 @@
(rec { const_ = x: _: x; a = const_ 1 a; }).a

6
samples/attrsets.nix

@ -0,0 +1,6 @@
{
a = 3;
b = a;
c.d = { e = 5; };
${c} = { f = 4; };
}

4
samples/complex-rec-scope.nix

@ -0,0 +1,4 @@
(rec {
value = 12;
func = (arg: value) 0;
}).func

1
samples/sample-function.nix

@ -0,0 +1 @@
{ foo, bar }@baz: bar

6
src/astformer/actions/consume-node.js

@ -0,0 +1,6 @@
"use strict";
// NOTE: This marker differs from RemoveNode in that it *doesn't* wipe out the state collected by the removed node; that is, it is assumed that the node is "consumed" and the stateLog is the result of that consumption. This is useful for various "meta-operations" which just serve to annotate some other operation with a modifier, and where the meta-operations themselves do not have any representation in the resulting query. In those cases, the meta-operation would be consumed and the parent node updated to reflect the modifier.
// FIXME: Check for existing places in optimizers where nodes are currently left lingering around, that should be consumed instead
module.exports = Symbol("ConsumeNode");

3
src/astformer/actions/no-change.js

@ -0,0 +1,3 @@
"use strict";
module.exports = Symbol("NoChange");

3
src/astformer/actions/remove-node.js

@ -0,0 +1,3 @@
"use strict";
module.exports = Symbol("RemoveNode");

20
src/astformer/combine-optimizers.js

@ -0,0 +1,20 @@
"use strict";
module.exports = function combineOptimizers(optimizers) {
let allVisitors = {};
for (let optimizer of optimizers) {
for (let [ key, visitor ] of Object.entries(optimizer.visitors)) {
if (allVisitors[key] == null) {
allVisitors[key] = [];
}
allVisitors[key].push({
name: optimizer.name,
func: visitor
});
}
}
return allVisitors;
};

16
src/astformer/create-debuggers.js

@ -0,0 +1,16 @@
"use strict";
const debug = require("debug");
module.exports = function createDebuggers(optimizers) {
let debuggers = {};
for (let optimizer of optimizers) {
debuggers[optimizer.name] = debug(`astformer:${optimizer.name}`);
debuggers[`${optimizer.name} (deferred)`] = debug(`astformer:${optimizer.name} (deferred)`);
}
debuggers["(subtree change)"] = debug(`astformer:(subtree change)`);
return debuggers;
};

27
src/astformer/handler-tracker.js

@ -0,0 +1,27 @@
"use strict";
module.exports = function createHandlerTracker() {
let handlers = new Map();
return {
add: function (name, func) {
if (!handlers.has(name)) {
handlers.set(name, []);
}
handlers.get(name).push(func);
},
call: function (name, value) {
let funcs = handlers.get(name);
if (funcs != null) {
for (let func of funcs) {
func(value);
}
}
},
has: function (name) {
return handlers.has(name);
}
};
};

361
src/astformer/index.js

@ -0,0 +1,361 @@
/* eslint-disable no-loop-func */
"use strict";
// Design note: We return stateLogs instead of passing in an object of registered handlers to call, because a node can become obsolete in mid-processing, and in those cases all of its state sets should be ignored. By far the easiest way to implement this, is to just keep a stateLog in the node handling context (since that entire context gets thrown away when processing gets aborted due to a subtree change), and let the parent deal with actually applying any still-relevant setStates to the correct handler functions.
// TODO: Figure out a way to track 'loss factor' per optimizer, ie. how many (partial or complete) node evaluations have been discarded due to the actions of that optimizer, including subtrees. This can give insight into which optimizers cause unreasonably much wasted work.
const util = require("util");
const splitFilter = require("split-filter");
const mapObj = require("fix-esm").require("map-obj").default;
const defaultValue = require("default-value");
const isPlainObj = require("fix-esm").require("is-plain-obj").default;
const findLast = require("find-last");
const NoChange = require("./actions/no-change");
const RemoveNode = require("./actions/remove-node");
const ConsumeNode = require("./actions/consume-node");
const typeOf = require("./util/type-of");
const concat = require("./util/concat");
const merge = require("./util/merge");
const measureTime = require("./util/measure-time");
const unreachable = require("@joepie91/unreachable")("jsnix");
const createHandlerTracker = require("./handler-tracker");
const createTimings = require("./timings-tracker");
const combineOptimizers = require("./combine-optimizers");
const createDebuggers = require("./create-debuggers");
const assureArray = require("assure-array");
const AnyChild = Symbol("AnyChild");
// FIXME: Implement a scope tracker of some sort, to decouple the code here a bit more
// TODO: Determine if we can improve performance by avoiding a lot of array allocations for the path tracking; by eg. nesting objects instead and unpacking it into an array on-demand
// FIXME: Verify that the various iterations=0 arguments are actually correct, and don't lose iteration count metadata
let EVALUATION_LIMIT = 10;
function defer(func) {
return { __type: "defer", func: func };
}
function handleNodeChildren(node, handleASTNode, path, originalContext, contextOverrides) {
let changedProperties = {};
let stateLogs = [];
function tryTransformItem(node, path, context) {
// console.log("--- PASSING IN", { context });
// console.log({path});
if (node == null) {
return node;
// } else if (node.__raqbASTNode === true) {
} else if (isPlainObj(node)) {
// FIXME: Is it correct to not specify an initialStateLog here?
let result = handleASTNode(node, 0, path, undefined, context);
if (result.stateLog.length > 0) {
stateLogs.push(result.stateLog);
}
return result.node;
} else if (Array.isArray(node)) {
let valuesHaveChanged = false;
let transformedArray = node.map((value, i) => {
let pathSegment = { type: "$array", key: i };
let transformedValue = tryTransformItem(value, path.concat([ pathSegment ]), context);
if (transformedValue !== value) {
valuesHaveChanged = true;
}
return transformedValue;
});
if (valuesHaveChanged) {
return transformedArray;
} else {
return node;
}
// } else if (isPlainObj(node)) {
// let newObject = {};
// let propertiesHaveChanged = false;
// for (let [ key, value ] of Object.entries(node)) {
// let pathSegment = { type: "$object", key: key };
// let transformedValue = tryTransformItem(value, path.concat([ pathSegment ]));
// if (transformedValue !== value) {
// propertiesHaveChanged = true;
// }
// newObject[key] = transformedValue;
// }
// if (propertiesHaveChanged) {
// return newObject;
// } else {
// return node;
// }
} else {
// Probably some kind of literal value; we don't touch these.
return node;
}
}
// FIXME: Delete nulls?
for (let [ property, value ] of Object.entries(node)) {
let childPath = path.concat([{ type: node.type, key: property }]);
let newContext = mergeContexts(originalContext, contextOverrides, property);
// console.log("--- MERGE", { newContext, property, originalContext, contextOverrides });
let transformedValue = tryTransformItem(value, childPath, newContext);
if (transformedValue !== value) {
changedProperties[property] = transformedValue;
}
}
return {
changedProperties: changedProperties,
stateLog: concat(stateLogs)
};
}
function mergeContexts(oldContext, overrides, property) {
let propertyOverrides = overrides[property];
let globalOverrides = overrides[AnyChild];
if (propertyOverrides == null && globalOverrides == null) {
// No changes
return oldContext;
} else {
// console.log("--- MERGING!", { oldContext, globalOverrides, propertyOverrides });
return {
... oldContext,
... globalOverrides ?? {},
... propertyOverrides ?? {}
};
}
}
module.exports = function optimizeTree(ast, optimizers) {
let debuggers = createDebuggers(optimizers);
let visitors = combineOptimizers(optimizers);
let timings = createTimings(optimizers);
let visitorsByType = mapObj(visitors, (key, value) => {
return [
key,
concat([
defaultValue(value, []),
defaultValue(visitors["*"], []),
])
];
});
function handleASTNode(node, iterations = 0, path = [], initialStateLog, context = {}) {
console.log({ path: path.map((item) => String(item.type)).join(" -> "), context });
// console.log({ path: path.map((item) => String(item.key)).join(" -> "), context });
// console.log(path.map((item) => String(item.key)).join(" -> "));
// The stateLog contains a record of every setState call that was made during the handling of this node and its children. We keep a log for this rather than calling handlers directly, because setState calls should always apply to *ancestors*, not to the current node. That is, if the current node does a setState for `foo`, and also has a handler registered for `foo`, then that handler should not be called, but the `foo` handler in the *parent* node should be.
// FIXME: Scope stateLog entries by optimizer name? To avoid name clashes for otherwise similar functionality. Like when multiple optimizers track column names. Also do this for context values! Maybe an escape hatch to deliberately define/reference globals or keys for other optimizers.
let stateLog = [];
let contextOverrides = {};
let defers = [];
let handlers = createHandlerTracker();
let nodeVisitors = visitorsByType[node.type];
function handleResult({ debuggerName, result, permitDefer, initialStateLog }) {
if (result === NoChange) {
// no-op
} else if (result == null) {
// FIXME: Figure out a better way to indicate the origin of such an issue, than the current error message format?
// FIXME: Include information on which node this failed for
throw new Error(`[${debuggerName}] A visitor is not allowed to return null or undefined; if you intended to leave the node untouched, return a NoChange marker instead`);
} else if (result === RemoveNode) {
debuggers[debuggerName](`Node of type '${typeOf(node)}' removed`);
return { node: RemoveNode, stateLog: [] };
} else if (result === ConsumeNode) {
debuggers[debuggerName](`Node of type '${typeOf(node)}' consumed, but its stateLog was left intact`);
stateLog.forEach((item) => { item.isFromConsumedNode = true; }); // NOTE: Mutates!
return { node: ConsumeNode, stateLog: stateLog };
} else if (result.__type === "defer") {
if (permitDefer) {
debuggers[debuggerName](`Defer was scheduled for node of type '${typeOf(node)}'`);
defers.push({ debuggerName, func: result.func });
} else {
throw new Error(`Cannot schedule a defer from within a defer handler`);
}
} else /*if (result.__raqbASTNode === true)*/ {
if (result === node) {
// Visitor returned the original node again; but in this case, it should return NoChange instead. We enforce this because after future changes to the optimizer implementation (eg. using an internally-mutable deep copy of the tree), we may no longer be able to *reliably* detect when the original node is returned; so it's best to already get people into the habit of returning a NoChange marker in those cases, by disallowing this.
throw new Error(`Visitor returned original node, but this may not work reliably; if you intended to leave the node untouched, return a NoChange marker instead`);
} else {
debuggers[debuggerName](`Node of type '${typeOf(node)}' replaced by node of type '${typeOf(result)}'`);
if (iterations >= EVALUATION_LIMIT) {
throw new Error(`Exceeded evaluation limit in optimizer ${debuggerName}; aborting optimization. If you are a user of raqb, please report this as a bug. If you are writing an optimizer, make sure that your optimizer eventually stabilizes on a terminal condition (ie. NoChange)!`);
} else {
return handleASTNode(result, iterations + 1, path, initialStateLog, context);
}
}
// } else {
// throw new Error(`Visitor returned an unexpected type of return value: ${util.inspect(result)}`);
}
}
function handleStateLog(newStateLog) {
let [ relevantState, otherState ] = splitFilter(newStateLog, (entry) => handlers.has(entry.name));
stateLog = stateLog.concat(otherState);
for (let item of relevantState) {
// FIXME: Log these, and which visitor they originate from
handlers.call(item.name, item.value);
}
}
function applyVisitorFunction({ visitorName, func, node, permitDefer }) {
let { value: result, time } = measureTime(() => {
return func(node, {
// eslint-disable-next-line no-loop-func
setState: (name, value) => {
// FIXME: util.inspect is slow, and not necessary when debug mode is disabled
debuggers[visitorName](`Setting state for '${name}' from node of type '${typeOf(node)}': ${util.inspect(value, { colors: true })}`);
stateLog.push({ name, value });
},
registerStateHandler: (name, func) => handlers.add(name, func),
defer: (permitDefer === true) ? defer : null,
findNearestStep: function (type) {
return (type != null)
? findLast(path, (item) => item.type === type)
: path[path.length - 1];
},
setContext: (children, key, value) => {
// FIXME: Turn this into an abstraction
// FIXME: Disallow this once we are in a `defer`; using it there is a bug, as child nodes have already been processed, and so context cannot be propagated to them anymore. Should throw an error telling the user that they probably have a bug in their code.
function setOne(child, key, value) {
if (contextOverrides[child] == null) {
contextOverrides[child] = {};
}
contextOverrides[child][key] = value;
}
if (children != null) {
assureArray(children).forEach((child) => {
setOne(child, key, value);
});
} else {
setOne(AnyChild, key, value);
}
},
getContext: (key) => {
// NOTE: We *do not* consider contextOverrides here. A node cannot set context for itself, only for its children. Instead, contextOverrides gets handled when passing a new context object to a child node upon its evaluation.
if (context[key] != null) {
return context[key];
} else {
throw new Error(`No key '${key}' exists in the context here`);
}
}
});
});
timings[visitorName] += time;
return result;
}
if (nodeVisitors != null) {
for (let visitor of nodeVisitors) {
let handled = handleResult({
debuggerName: visitor.name,
result: applyVisitorFunction({
visitorName: visitor.name,
func: visitor.func,
node: node,
permitDefer: true
}),
permitDefer: true
});
if (handled != null) {
// Handling of the current node was aborted
return handled;
}
}
}
let childResult = handleNodeChildren(node, handleASTNode, path, context, contextOverrides);
if (Object.keys(childResult.changedProperties).length > 0) {
let newNode = merge(node, childResult.changedProperties);
// We already know that the new node is a different one, but let's just lead it through the same handleResult process, for consistency. Handling of the pre-child-changes node is aborted here, and we re-evaluate with the new node.
let reevaluatedResult = handleResult({
debuggerName: "(subtree change)",
result: newNode,
permitDefer: false,
// NOTE: If we have any leftover state from nodes that were consumed upstream, we should make sure to include this in the reevaluation, even when the subtree was replaced!
initialStateLog: (childResult.stateLog.length > 0)
? childResult.stateLog.filter((item) => item.isFromConsumedNode)
: undefined
});
return reevaluatedResult;
}
if (initialStateLog != null) {
// NOTE: We intentionally process the initialStateLog here and not earlier; that way it is consistent with how any retained stateLog entries *would* have executed on the node before it got replaced (ie. after evaluation of the children). Conceptually you can think of it as the initialStateLog being prefixed to the stateLog of the childResult.
handleStateLog(initialStateLog);
}
if (childResult.stateLog.length > 0) {
handleStateLog(childResult.stateLog);
}
for (let defer of defers) {
let handled = handleResult({
debuggerName: `${defer.debuggerName} (deferred)`,
result: applyVisitorFunction({
visitorName: defer.debuggerName,
func: defer.func,
node: node,
permitDefer: false
}),
permitDefer: false
});
if (handled != null) {
// Handling of the current node was aborted
return handled;
}
}
return {
stateLog: stateLog,
node: node
};
}
let { value: rootResult, time } = measureTime(() => {
return handleASTNode(ast);
});
let timeSpentInOptimizers = Object.values(timings).reduce((sum, n) => sum + n, 0);
if (rootResult.node !== RemoveNode && rootResult.node !== ConsumeNode) {
return {
ast: rootResult.node,
timings: {
"# Total": time,
"# Walker overhead": time - timeSpentInOptimizers,
... timings,
}
};
} else {
unreachable("Root node was removed");
}
};

12
src/astformer/timings-tracker.js

@ -0,0 +1,12 @@
"use strict";
module.exports = function createTimings(optimizers) {
let timings = {};
for (let optimizer of optimizers) {
// timings[optimizer.name] = 0n;
timings[optimizer.name] = 0;
}
return timings;
};

11
src/astformer/util/concat.js

@ -0,0 +1,11 @@
"use strict";
module.exports = function concat(arrays) {
if (arrays.length === 0) {
return [];
} else if (arrays.length === 1) {
return arrays[0];
} else {
return arrays[0].concat(... arrays.slice(1));
}
};

19
src/astformer/util/measure-time.js

@ -0,0 +1,19 @@
"use strict";
function hrtimeToNanoseconds(time) {
// If the numbers here become big enough to cause loss of precision, we probably have bigger issues than numeric precision...
return (time[0] * 1e9) + time[1];
}
module.exports = function measureTime(func) {
// let startTime = process.hrtime.bigint();
let startTime = hrtimeToNanoseconds(process.hrtime());
let result = func();
// let endTime = process.hrtime.bigint();
let endTime = hrtimeToNanoseconds(process.hrtime());
return {
value: result,
time: (endTime - startTime)
};
};

5
src/astformer/util/merge.js

@ -0,0 +1,5 @@
"use strict";
module.exports = function merge(... items) {
return Object.assign({}, ... items);
};

12
src/astformer/util/type-of.js

@ -0,0 +1,12 @@
"use strict";
module.exports = function typeOf(value) {
// FIXME: Better check
if (value == null) {
return null;
} else if (typeof value === "object") {
return value.type;
} else {
return null;
}
};

14
src/parse.js

@ -0,0 +1,14 @@
"use strict";
const Parser = require("tree-sitter");
const Nix = require("tree-sitter-nix");
const prepareAst = require("./prepare-ast");
module.exports = function parseNix(source) {
// TODO: Can we reuse the parser instance?
let parser = new Parser();
parser.setLanguage(Nix);
return prepareAst(parser.parse(source));
};

131
src/prepare-ast.js

@ -0,0 +1,131 @@
"use strict";
const matchValue = require("match-value");
const asExpression = require("as-expression");
function convertStringNodeParts(node) {
let fullText = node.text;
let currentIndex = node.startIndex;
let parts = [];
function relative(index) {
return index - node.startIndex;
}
for (let child of node.children) {
if (child.type === "interpolation") {
if (child.startIndex > currentIndex) {
// We skipped some literal string
let value = fullText.slice(relative(currentIndex), relative(child.startIndex));
parts.push({ type: "NixInterpolationLiteral", value: value });
}
parts.push(convertNode(child));
currentIndex = child.endIndex;
}
}
if (currentIndex < node.endIndex) {
// Last bit of string literal at the end
let value = fullText.slice(relative(currentIndex), relative(node.endIndex));
parts.push({ type: "NixInterpolationLiteral", value: value });
}
return parts;
}
function convertNode(node) {
let { type } = node;
let result = (node.isNamed)
? { type: type }
: { type: "token", text: node.text }
if (node.fields != null) {
for (let field of node.fields) {
let children = node[field];
let fieldName = field.replace(/Nodes?$/, "");
result[fieldName] = asExpression(() => {
if (children == null) {
return null;
} else if (Array.isArray(children)) {
return children.map(convertNode);
} else {
return convertNode(children);
}
});
}
}
// Special case: if we don't provide a Babel-compatible top-level Program entry, traversal will fail
if (type === "source_expression") {
return {
type: "Program",
sourceType: "script",
body: [ result.expression ]
};
}
// The below section is based on `alias` expressions throughout the grammar, and the rules in https://github.com/cstrahan/tree-sitter-nix/blob/83ee5993560bf15854c69b77d92e34456f8fb655/grammar.js#L53-L59
if (type === "identifier" || type === "attr_identifier") {
result.name = node.text;
} else if (type === "integer" || type === "float") {
result.value = node.text;
} else if (type === "string") {
result.parts = convertStringNodeParts(node);
} else if (type === "path" || type === "hpath") {
result.path = node.text;
} else if (type === "spath") {
// Strip the < and >
result.path = node.text.slice(1, -1);
} else if (type === "uri") {
result.uri = node.text;
}
if (type === "binary") {
// Unpack the anonymous token
result.operator = result.operator.text;
}
if (type === "rec_attrset") {
result.recursive = true;
} else if (type === "attrset") {
result.recursive = false;
}
result.type = matchValue(result.type, {
source_expression: "NixProgram",
token: "NixAnonymousToken",
path: "NixPathLiteral",
hpath: "NixHomePathLiteral",
spath: "NixEnvironmentPathLiteral",
uri: "NixURILiteral",
integer: "NixIntegerLiteral",
float: "NixFloatLiteral",
string: "NixStringLiteral",
parenthesized: "NixParenthesizedExpression",
attrset: "NixAttributeSet",
rec_attrset: "NixAttributeSet",
identifier: "NixIdentifier",
attr_identifier: "NixAttributeIdentifier",
attrpath: "NixAttributePath",
bind: "NixBinding",
binary: "NixBinaryOperation",
app: "NixFunctionCall",
select: "NixAttributeSelection",
interpolation: "NixInterpolationExpression",
// Function definitions
function: "NixFunctionDefinition",
formals: "NixUnpackedAttributes",
formal: "NixUnpackedAttribute",
});
return result;
}
/* Produces an AST that's roughly shape-compatible with ESTree so that Babel can deal with it */
module.exports = function prepareAST(tree) {
return convertNode(tree.rootNode);
};

39
src/print-ast.js

@ -0,0 +1,39 @@
"use strict";
const chalk = require("chalk");
const util = require("util");
const INDENT = chalk.gray("│") + " ";
function formatProperties(node) {
let relevantProperties = Object.keys(node)
.filter((property) => property !== "type" && typeof node[property] !== "object");
return relevantProperties
.map((property) => `${chalk.yellow.bold(property)}: ${util.inspect(node[property], { colors: true })}`)
.join(" ");
}
function printNode(node, level) {
console.log(INDENT.repeat(level) + chalk.bold(node.type ?? "") + " " + formatProperties(node));
for (let [ key, value ] of Object.entries(node)) {
if (typeof value === "object" && value != null) {
let isArray = Array.isArray(value);
console.log(INDENT.repeat(level + 1) + chalk.cyan(key) + ":" + (isArray ? chalk.gray(" []") : ""));
if (isArray) {
value.forEach((item) => printNode(item, level + 1));
} else {
printNode(value, level + 1);
}
}
}
}
// TODO: Add a non-printing renderAST method, for embedding in existing console.* calls
module.exports = function printAST(ast) {
return printNode(ast, 0);
};

44
src/transformers/_nix-types.js

@ -0,0 +1,44 @@
"use strict";
const assert = require("assert");
let types = module.exports = {
/* NOTE: Synthesized attribute sets are non-recursive:
nix-repl> rec { data = 1; sub = { data = 2; }; sub.ref = data; }.sub.ref
1
nix-repl> rec { data = 1; sub = rec { data = 2; }; sub.ref = data; }.sub.ref
2
nix-repl> rec { data = 1; sub.data = 2; sub.ref = data; }.sub.ref
1
*/
NixAttributeSet: function (bindings, recursive = false) {
return {
type: "NixAttributeSet",
bind: bindings,
recursive: recursive
};
},
NixBinding: function (attributePath, value) {
return {
type: "NixBinding",
attrpath: types.NixAttributePath(attributePath),
expression: value
};
},
NixAttributePath: function (attributes) {
assert(Array.isArray(attributes));
return {
type: "NixAttributePath",
attr: attributes
};
},
NixAttributeIdentifier: function (name) {
return {
type: "NixAttributeIdentifier",
name: name
};
}
};

9
src/transformers/_unpack-expression.js

@ -0,0 +1,9 @@
"use strict";
const assert = require("assert");
// This utility function serves to unpack an expression from an ExpressionStatement node; in many cases, @babel/template insists on wrapping every expression in such a statement.
module.exports = function (node) {
assert(node.type === "ExpressionStatement");
return node.expression;
};

100
src/transformers/attribute-sets.js

@ -0,0 +1,100 @@
"use strict";
const assert = require("assert");
const types = require("@babel/types");
const template = require("@babel/template").default;
const unpackExpression = require("./_unpack-expression");
const NoChange = require("../astformer/actions/no-change");
// FIXME: Add expression parens!
let tmplCallLazy = template(`
%%wrapper%%()
`);
let tmplLazyWrapper = template(`(
() => %%expression%%
)`);
let tmplLazyWrapperRecursive = template(`(
function() { return %%expression%%; }
)`);
let tmplExtend = template(`(
$$jsNix$extend(this ?? {}, %%object%%)
)`);
function lazyEvaluationWrapper(args) {
let { recursive, ... rest } = args;
let wrapper = (recursive)
? tmplLazyWrapperRecursive
: tmplLazyWrapper;
return unpackExpression(wrapper(rest));
}
function isDynamicBinding(binding) {
return binding.attrpath.attr[0].type !== "NixAttributeIdentifier";
}
module.exports = {
name: "attribute-sets",
visitors: {
Program: (_node, { setContext }) => {
setContext(null, "attributeSets_inRecursiveSet", false);
return NoChange;
},
NixAttributeSelection: (_node, { defer }) => {
// TODO: Optimize dynamic attribute access by translating builtins.getAttr to dynamic property access at compile time? This requires scope analysis!
return defer((node) => {
assert(node.attrpath.type === "NixAttributePath");
return node.attrpath.attr.reduce((last, identifier) => {
assert(identifier.type === "NixAttributeIdentifier");
return unpackExpression(tmplCallLazy({
wrapper: types.memberExpression(last, types.identifier(identifier.name))
}));
}, node.expression);
});
},
NixAttributeSet: (node, { defer, setContext, getContext }) => {
if (node.recursive) {
setContext(null, "attributeSets_inRecursiveSet", true);
}
return defer((node) => {
let isRecursive = node.recursive;
let hasDynamicBindings = node.bind.some((binding) => isDynamicBinding(binding));
if (hasDynamicBindings) {
throw new Error(`UNIMPLEMENTED: Dynamic bindings are not supported yet`);
} else {
let object = types.objectExpression(node.bind.map((binding) => {
assert(binding.attrpath.attr.length === 1); // Nested attributes should have been desugared by this point
let name = binding.attrpath.attr[0].name;
let expression = binding.expression;
return types.objectProperty(
types.stringLiteral(name),
lazyEvaluationWrapper({ recursive: isRecursive, expression: expression })
);
}));
// FIXME/MARKER: This is currently broken!
console.log({context: getContext("attributeSets_inRecursiveSet")});
// Only do prototypal inheritance for `this` if we're dealing with nested recursive attribute sets
if (isRecursive && getContext("attributeSets_inRecursiveSet") === true) {
return unpackExpression(tmplExtend({
object: object
}));
} else {
return object;
}
}
});
},
}
};

91
src/transformers/desugar-attrsets.js

@ -0,0 +1,91 @@
"use strict";
const NoChange = require("../astformer/actions/no-change");
const { NixAttributeIdentifier, NixAttributeSet, NixBinding } = require("./_nix-types");
function isAttributeSet(node) {
return (node.type === "NixAttributeSet");
}
function mergeAttributeSets(a, b) {
return NixAttributeSet([ ... a.bind, ... b.bind ]);
}
function mergeBindings(name, a, b) {
let attributes = mergeAttributeSets(unpackBindingValue(a), unpackBindingValue(b));
return NixBinding(
[ NixAttributeIdentifier(name) ],
attributes
);
}
function unpackBindingValue(binding) {
// binding -> value for top-level attribute of the binding
if (binding.attrpath.attr.length > 1) {
// Nested attribute path syntax, synthesize a new attribute set with the first attribute snipped off - this new attribute set represents the *value* of the binding
return NixAttributeSet([
NixBinding(binding.attrpath.attr.slice(1), binding.expression)
]);
} else {
return binding.expression;
}
}
module.exports = {
name: "desugar-attrsets",
visitors: {
NixAttributeSet: (node) => {
let neededDesugaring = false;
let newStaticBindings = {};
let dynamicBindings = [];
for (let binding of node.bind) {
if (binding.attrpath.attr.length > 1) {
neededDesugaring = true;
}
let firstAttribute = binding.attrpath.attr[0];
let isStaticAttribute = firstAttribute.type === "NixAttributeIdentifier";
if (isStaticAttribute) {
let attributeName = firstAttribute.name;
let existingBinding = newStaticBindings[attributeName];
if (existingBinding == null) {
newStaticBindings[attributeName] = NixBinding(
[ firstAttribute ],
unpackBindingValue(binding)
);
} else {
if (isAttributeSet(existingBinding.expression) && isAttributeSet(binding.expression)) {
neededDesugaring = true;
newStaticBindings[attributeName] = mergeBindings(attributeName, existingBinding, binding);
} else {
throw new Error(`Key '${attributeName}' was specified twice, but this is only allowed when both values are an attribute set`);
}
}
} else {
// FIXME: Needs runtime check, need to *always* construct objects at runtime when dynamic bindings are involved
dynamicBindings.push(binding);
}
}
let staticBindingList = Object.entries(newStaticBindings).map(([ key, value ]) => {
// console.log([ key, value ]);
return value;
});
// We only check this here because there are multiple things that could cause a need for desugaring, and this is simpler for now than having a separate 'check first' implementation
if (neededDesugaring) {
return NixAttributeSet([ ... staticBindingList, ... dynamicBindings ]);
} else {
return NoChange;
}
}
}
};

110
src/transformers/functions.js

@ -0,0 +1,110 @@
"use strict";
const asExpression = require("as-expression");
const assert = require("assert");
const types = require("@babel/types");
const template = require("@babel/template").default;
const unpackExpression = require("./_unpack-expression");
// TODO: Memoize every function
// NOTE: These are arrow functions because variable references within functions should always refer to the nearest scope; and since we use `this` to handle variable references within recursive attribute sets, we need to ensure that a function definition *does not* create its own `this` context.
let tmplFunctionDefinitionFormalsUniversal = template(`
((%%universal%%) => {
const %%formals%% = %%universal%%;
return %%body%%;
})
`);
// FIXME: Test that this template form actually works with our formals structure
let tmplFunctionDefinitionFormals = template(`
((%%formals%%) => {
return %%body%%;
})
`);
let tmplFunctionDefinitionUniversal = template(`
((%%universal%%) => {
return %%body%%;
})
`);
let tmplFunctionCall = template(`
(%%function%%(%%arg%%))
`);
function functionDefinition({ universal, formals, body }) {
let convertedFormals = asExpression(() => {
if (formals != null) {
return types.objectPattern(formals.map((formal) => {
return types.objectProperty(
types.identifier(formal),
types.identifier(formal)
);
}));
} else {
return undefined;
}
});
if (universal != null && formals != null) {
return tmplFunctionDefinitionFormalsUniversal({
body: body,
universal: universal,
formals: convertedFormals
});
} else if (universal != null) {
return tmplFunctionDefinitionUniversal({
body: body,
universal: universal
});
} else {
return tmplFunctionDefinitionFormals({
body: body,
formals: convertedFormals
});
}
}
module.exports = {
name: "functions",
visitors: {
NixFunctionDefinition: (_node, { defer, setContext }) => {
setContext([ "universal", "formals" ], "identifierType", "define");
return defer((node) => {
return unpackExpression(
functionDefinition({
universal: asExpression(() => {
if (node.universal != null) {
assert(node.universal.type === "Identifier");
return node.universal.name;
}
}),
formals: asExpression(() => {
if (node.formals != null) {
assert(node.formals.type === "NixUnpackedAttributes");
return node.formals.formal.map((formal) => {
assert(formal.type === "NixUnpackedAttribute");
assert(formal.name.type === "Identifier");
return formal.name.name;
});
}
}),
body: node.body
})
);
});
},
NixFunctionCall: (_node, { defer }) => {
return defer((node) => {
return unpackExpression(
tmplFunctionCall({
function: node.function,
arg: node.argument
})
);
});
}
}
};

108
src/transformers/index.js

@ -0,0 +1,108 @@
"use strict";
const assert = require("assert");
const types = require("@babel/types");
const template = require("@babel/template").default;
const unreachable = require("@joepie91/unreachable")("jsnix");
const NoChange = require("../astformer/actions/no-change");
const ConsumeNode = require("../astformer/actions/consume-node");
const RemoveNode = require("../astformer/actions/remove-node");
const asExpression = require("as-expression");
const unpackExpression = require("./_unpack-expression");
const printAst = require("../print-ast");
// FIXME: Make strict mode! Otherwise objects will inherit from `global`
let tmplModule = template(`
module.exports = function({ builtins, $$jsNix$extend }) {
return %%contents%%;
};
`);
let tmplLetIn = template(`
(() => {
%%letBindings%%
return %%expression%%;
})()
`);
let tmplObjectDynamic = template(`
(() => {
let $$nixJS_object = {};
%%entries%%
return $$nixJS_object;
})()
`);
let tmplObjectDynamicEntry = template(`
if ($$nixJS_object[%%key%%] !== undefined) { throw new Error(\`Duplicate key '\${%%key%%}' in attribute set\`); }
$$nixJS_object[%%key%%] = %%wrappedExpression%%;
`);
let tmplIdentifierReference = template(`(
(this?.%%name%% ?? %%name%%)()
)`);
function generateDynamicObject(bindings, recursive) {
let generatedBindings = bindings.map((binding) => {
let wrapper = (recursive)
? tmplLazyWrapperRecursive
: tmplLazyWrapper;
let key = isDynamicBinding(binding)
? binding.attrpath.attr[0]
: binding.attrpath.attr[0].name;
return tmplObjectDynamicEntry({
key: key,
wrappedExpression: wrapper({ expression: binding.expression })
});
});
return tmplObjectDynamic({ entries: generatedBindings.flat() })
}
let trivial = {
name: "trivial-transforms",
visitors: {
// Trivial transforms
NixParenthesizedExpression: (node) => node.expression,
Program: (_node, { setContext, defer }) => {
setContext(null, "identifierType", "reference");
return defer((node) => {
assert(node.body.length === 1);
return tmplModule({ contents: node.body[0] });
});
},
NixIdentifier: (node, { getContext }) => {
// FIXME: Mangle reserved keywords like `const`
if (getContext("identifierType") === "define") {
return types.identifier(node.name);
} else { // reference
return unpackExpression(tmplIdentifierReference({ name: node.name }));
}
},
NixBinaryOperation: (_node, { defer }) => {
return defer((node) => {
// FIXME: Verify that all the 'operator' values match between Nix and JS!
return types.binaryExpression(node.operator, node.left, node.right);
});
}
}
};
module.exports = [
require("./desugar-attrsets"),
require("./literals"),
require("./functions"),
require("./attribute-sets"),
trivial,
];

11
src/transformers/literals.js

@ -0,0 +1,11 @@
"use strict";
const types = require("@babel/types");
module.exports = {
name: "literals",
visitors: {
NixIntegerLiteral: (node) => types.numericLiteral(parseInt(node.value)),
NixFloatLiteral: (node) => types.numericLiteral(parseFloat(node.value)),
}
};

45
src/transpile.js

@ -0,0 +1,45 @@
"use strict";
const astformer = require("./astformer");
const parse = require("./parse");
const printAST = require("./print-ast");
const transformers = require("./transformers");
const generate = require("@babel/generator").default;
module.exports = function transpile(nixSource) {
if (process.env.DEBUG_NIX) {
console.log("-- DEBUG (SOURCE):\n", nixSource);
}
let ast = parse(nixSource);
if (process.env.DEBUG_NIX) {
console.log("-- DEBUG (AST):");
printAST(ast);
}
// Warm-up for hot VM performance testing
// for (let i = 0; i < 10000; i++) {
// astformer(ast, transformers);
// }
let result = astformer(ast, transformers);
if (process.env.DEBUG_NIX) {
console.log("-- DEBUG (TRANSPILED AST):");
printAST(result.ast);
console.log({ timings: result.timings });
}
let code = generate(result.ast).code;
if (process.env.DEBUG_NIX) {
console.log("-- DEBUG (INPUT):");
console.log(nixSource);
console.log("-- DEBUG (OUTPUT):");
console.log(code);
}
return code;
};

12
testers/parse.js

@ -0,0 +1,12 @@
"use strict";
const fs = require("fs");
const assert = require("assert");
const parse = require("../src/parse");
const printAST = require("../src/print-ast");
assert(process.argv[2] != null);
let tree = parse(fs.readFileSync(process.argv[2], "utf8"));
// console.dir(tree, { depth: null })
printAST(tree);

14
testers/transform.js

@ -0,0 +1,14 @@
"use strict";
const fs = require("fs");
const assert = require("assert");
const parse = require("../src/parse");
const astformer = require("../src/astformer");
const transformers = require("../src/transformers");
const printAST = require("../src/print-ast");
assert(process.argv[2] != null);
let tree = parse(fs.readFileSync(process.argv[2], "utf8"));
let transformed = astformer(tree, transformers);
printAST(transformed);

19
todo-wrong.txt

@ -0,0 +1,19 @@
NOTE: This is wrong. It checks `this.x ?? x` but `x` should have precedence over `this.x` since it was defined later!
module.exports = function ({
builtins,
$$jsNix$extend
}) {
return {
"const_": function () {
return x => {
return _ => {
return (this?.x ?? x)();
};