Sven Slootweg 7 months ago
parent
commit
bfe0422ff6
  1. BIN
      OrWC7QSKCF8BDFRaRaj0Z4ul.png
  2. BIN
      eXPynWGvSXtvf3g50oWWXDhq.png
  3. 100
      notes.txt
  4. 3
      package.json
  5. 3
      roadmap.txt
  6. 8
      samples/attrsets.nix
  7. 4
      samples/dynamic.nix
  8. 10
      shell.nix
  9. 2
      src/astformer/util/measure-time.js
  10. 15
      src/evaluate.js
  11. 2
      src/mangle-name.js
  12. 2
      src/prepare-ast.js
  13. 29
      src/print-ast.js
  14. 146
      src/transformers/attribute-sets.js
  15. 13
      src/transformers/desugar-attrsets.js
  16. 5
      src/transformers/desugar-inherits.js
  17. 27
      src/transformers/desugar-interpolation-expressions.js
  18. 4
      src/transformers/functions.js
  19. 3
      src/transformers/index.js
  20. 12
      src/transformers/literals.js
  21. 2
      src/transformers/templates/lazy-wrapper.js
  22. 15
      src/transformers/templates/object-literal.js
  23. 4
      src/transformers/util/nix-types.js
  24. 14
      src/transformers/util/template-expression.js

BIN
OrWC7QSKCF8BDFRaRaj0Z4ul.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

BIN
eXPynWGvSXtvf3g50oWWXDhq.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

100
notes.txt

@ -6,3 +6,103 @@ let a = 0; in rec { a = 1; foo = bar: a * 2 }
let a = 0; in rec { a = 1; foo = let func = bar: a * 2; in rec { a = 3; baz = func; }; }
# 2
----
Things that could inhibit evaluation of a thunk:
- Conditionals (already handled by JS semantics)
lazy-wrapping expressions which are object/attrset properties, array/list elements, function args, bindings(!), and so on (and unpacking them on the corresponding accesses) but otherwise just directly translating to JS
Note: equality comparison for lists and attrsets is a *deep* comparison by default! Need an equality operator wrapper to handle this. For derivations, equality is determined by the outPath. Functions are never equal, and the only type compatibility is between integers and floats.
----
General wrapping concept: expressions are wrapped in a lazy eval wrapper when they are either bindings or members of a complex data structure, where evaluating the data structure itself *does not* imply evaluating its members. The "set of function args in a function call" counts as a complex data structure.
Binding access can be blindly unwrapped because regardless of whether it's a local binding or eg. originates from a `with` statement (that takes an object), the binding itself will always contain a wrapped expression.
Passing around wrapped unevaluated expressions (eg. assigning an object property to *another* object property, or passing an object property into a function call) is made possible in a somewhat hacky way; instead of first unwrapping (evaluating) the source expression and then passing the result somewhere, the evaluation *itself* will (according to the above principles) be wrapped in a lazy wrapper, meaning that the evaluation of the source expression will only take place if the *new* wrapper is evaluated somewhere.
This means that you end up with something along the lines of `someFunc(() => otherWrapper())` - the immediate call from within a wrapper might *seem* redundant, but therefore isn't. It may be possible to improve on this in the future, but for now this significantly reduces compiler complexity, because the compiler does not need to track provenance of the source expression to determine when it can be passed in directly as-is, and does not need to do code analysis to determine whether it is safe to do so in all cases. Let's just let V8 worry about optimizing this for now.
Language elements:
- Number literal
- String literal
- Path literal
- URL literal
- Object literal (wrap object property values)
- Object literal, recursive (wrap object property values)
- Object literal, inherits (DO NOT wrap/unwrap, all inheritable values are already wrapped)
- Array literal (wrap array elements)
- let..in bindings (wrap binding values)
- Function definition (DO NOT wrap or unwrap args, this is handled at call/access sites)
- Function definition, @-all arg (DO NOT wrap/unwrap, all arguments are already wrapped, just convert to object literal)
- Binding reference (unwrap)
- Assert
- Conditional, if/then/else (DO NOT do extra wrapping/unwrapping, JS semantics take care of the conditional unwrapping here)
- `with` expression (DO NOT unwrap, just apply the properties as bindings directly in their already-wrapped form)
- Binop: Access property, exists (unwrap)
- Binop: Access property, does not exist (fatal)
- Binop: Access property with fallback, does not exist (DO NOT wrap/unwrap, this is just a conditional!)
- Binop: Call function (wrap arg expressions) -- note: SHOULD NOT evaluate inputs to those expressions at call time, see example below
- Binop: Arithmetic negate
- Binop: Has attribute (unwrap object properties for all path segments except the final one)
- Binop: Concatenate array (DO NOT unwrap array elements, concat them 'blindly')
- Binop: Multiply
- Binop: Divide
- Binop: Add
- Binop: Subtract
- Binop: Concatenate string
- Binop: Boolean negate
- Binop: Merge objects (DO NOT unwrap properties, merge them 'blindly')
- Binop: Less than
- Binop: Less than or equal to
- Binop: More than
- Binop: More than or equal to
- Binop: Is equal
- Binop: Is not equal
- Binop: Logical AND
- Binop: Logical OR
- Binop: Logical implication
Things that wrap:
- Object literal, explicit property values only, NOT inherited properties
- Array literal, elements
- Scope bindings (definitions)
- Function call arguments, named/single arguments only, NOT members of catch-all @
Things that unwrap:
- Binding access
- Object property access (including every property in a `has attribute` except for the last one)
- Array element access
////////
let arg = { a: trace "foo" 42 }
let func = stuff: trace "called" true
func trace "pass" arg.a
nix-repl> let arg = { a = builtins.trace "foo" 42; }; func = stuff: (builtins.trace "called" true); in func (builtins.trace "pass" arg.a)
trace: called
true
nix-repl> let arg = { a = builtins.trace "foo" 42; }; func = stuff: (builtins.trace "called" stuff); in func (builtins.trace "pass" arg.a)
trace: called
trace: pass
trace: foo
42
//////
maybe cases
rec { a = 1; b = 2; c = { inherit a; }; }.c
let a = "foo"; b = a; in { "${a}" = 1; "${b}" = 2; }
let a = "foo"; in { "${a}" = 1; "${a}" = 2; }
let a = "foo"; in { "${foo}" = 1; "${foo}" = 2; }
nix-repl> let a = "foo"; ${a} = "bar"; in true
error: dynamic attributes not allowed in let at (string):1:1

3
package.json

@ -4,7 +4,8 @@
"description": "",
"main": "index.js",
"scripts": {
"test": "tape tests/**/* | tap-difflet"
"test": "tape tests/**/* | tap-difflet",
"eval": "DEBUG_NIX=1 node run.js"
},
"repository": {
"type": "git",

3
roadmap.txt

@ -1,4 +1,7 @@
- stringify Nix
- convert rec attrset to use heap-of-vars
- wrap function args in lazy wrapper + unwrap identifier access?
- memoize functions
- let..in -> rec attrset
- rename 'mangling'
- verify that $ cannot appear in genuine Nix identifiers unescaped

8
samples/attrsets.nix

@ -1,6 +1,8 @@
{
a = 3;
rec {
a = "hi";
b = a;
c.d = { e = 5; };
${c} = { f = 4; };
"${b}s" = { f = 4; };
# FIXME: The below currently breaks desugar-attrsets
# ${c}.d = { g = 6; };
}

4
samples/dynamic.nix

@ -0,0 +1,4 @@
let key = "a"; in {
a = 5;
${key} = 3;
}.a

10
shell.nix

@ -0,0 +1,10 @@
let
pkgs = import <nixpkgs> { };
in
pkgs.mkShell {
packages = [
pkgs.nodejs
pkgs.nodejs.python
pkgs.yarn
];
}

2
src/astformer/util/measure-time.js

@ -1,6 +1,6 @@
"use strict";
// FIXME: Make separate package, measure-function-call or so
// FIXME: Replace with `time-call` package
function hrtimeToNanoseconds(time) {
// If the numbers here become big enough to cause loss of precision, we probably have bigger issues than numeric precision...

15
src/evaluate.js

@ -8,7 +8,7 @@ module.exports = function evaluate(nixCode) {
const api = {
builtins: {},
$$jsNix$memoize: function (func) {
$memoize: function (func) {
let isCalled = false;
let storedResult;
@ -21,7 +21,7 @@ module.exports = function evaluate(nixCode) {
return storedResult;
};
},
$$jsNix$handleArgument: function (name, _arg, defaultValue) {
$handleArgument: function (name, _arg, defaultValue) {
// We need to evaluate the lazy wrapper for `arg`, as it is passed in as an attribute set; and since that is a value, it will have been wrapped.
const arg = _arg();
@ -41,6 +41,17 @@ module.exports = function evaluate(nixCode) {
// FIXME: Improve error
throw new Error(`Must pass an attribute set to function that expects one`);
}
},
$assertUniqueKeys: function (keys) {
let seen = new Set();
for (let key of keys) {
if (seen.has(key)) {
throw new Error(`Attempted to define duplicate attribute '${key}'`);
} else {
seen.add(key);
}
}
}
};

2
src/mangle-name.js

@ -18,7 +18,7 @@ module.exports = function mangleName(name) {
assert(name.length > 0);
// FIXME: Tag an identifier of this type with an internal property instead
if (name.startsWith("$$jsNix$")) {
if (name.startsWith("$")) {
return name;
} else {
let completedFirstCharacter = false;

2
src/prepare-ast.js

@ -51,6 +51,8 @@ function convertStringNodeParts(node) {
}
return { ... part, value: strippedString };
} else {
return part;
}
});
}

29
src/print-ast.js

@ -15,20 +15,25 @@ function formatProperties(node) {
}
function printNode(node, level) {
console.log(INDENT.repeat(level) + chalk.bold(node.type ?? "") + " " + formatProperties(node));
for (let [ key, value ] of Object.entries(node)) {
if (typeof value === "object" && value != null) {
let isArray = Array.isArray(value);
console.log(INDENT.repeat(level + 1) + chalk.cyan(key) + ":" + (isArray ? chalk.gray(" []") : ""));
if (isArray) {
value.forEach((item) => printNode(item, level + 1));
} else {
printNode(value, level + 1);
if (node != null) {
console.log(INDENT.repeat(level) + chalk.bold(node.type ?? "") + " " + formatProperties(node));
for (let [ key, value ] of Object.entries(node)) {
if (typeof value === "object" && value != null) {
let isArray = Array.isArray(value);
console.log(INDENT.repeat(level + 1) + chalk.cyan(key) + ":" + (isArray ? chalk.gray(" []") : ""));
if (isArray) {
value.forEach((item) => printNode(item, level + 1));
} else {
printNode(value, level + 1);
}
}
}
} else {
// This should never happen!
console.log(INDENT.repeat(level) + chalk.bold.red(`! ${node}`));
}
}

146
src/transformers/attribute-sets.js

@ -6,18 +6,29 @@ const template = require("@babel/template").default;
const splitFilter = require("split-filter");
const unpackExpression = require("./util/unpack-expression");
const templateExpression = require("./util/template-expression");
const NoChange = require("../astformer/actions/no-change");
const printAst = require("../print-ast");
const lazyWrapper = require("./templates/lazy-wrapper");
const callLazyWrapper = require("./templates/call-lazy-wrapper");
const objectLiteral = require("./templates/object-literal");
// TODO: Optimize lazy evaluation wrappers by only unpacking them selectively when used in an actual expression; in particular, that avoids the "wrapper that just calls another wrapper" overhead when passing attributes as function arguments
// TODO: Change to a prototype-based scope object chain? This should produce more consistent output regardless of whether a given set of bindings is static vs. dynamic and recursive vs. non-recursive
let tmplExtend = template(`(
$$jsNix$extend(this ?? {}, %%object%%)
)`);
let tmplAssertKeys = template(`
$assertUniqueKeys( %%keyList%% )
`);
let tmplScopeWrapper = template(`(
let tmplObjectNormal = templateExpression(`
(() => {
%%keyAssertion%%;
return %%object%%;
})()
`);
let tmplScopeWrapper = templateExpression(`(
(() => {
%%bindings%%;
@ -25,6 +36,23 @@ let tmplScopeWrapper = template(`(
})()
)`);
let tmplDynamicScopeWrapper = templateExpression(`(
(() => {
%%keyAssertion%%;
let $attributes = {};
with ($attributes) {
/* Static and overrides */
Object.assign($attributes, {
%%bindings%%;
});
}
return $attributes;
})()
)`);
// FIXME: Verify that this always works, and that we don't need `var` for hoisting!
let tmplRecursiveBinding = template(`
const %%name%% = %%expression%%;
@ -35,6 +63,52 @@ function isDynamicBinding(binding) {
return binding.attrpath.attr[0].type !== "NixAttributeIdentifier";
}
function objectNormal(bindings) {
return tmplObjectNormal({
object: objectLiteral(bindings.map(({ name, expression }) => {
return [ name, expression ];
})),
keyAssertion: bindings.some((binding) => typeof binding.name !== "string")
// Only needed when dealing with dynamic keys
? assertKeys(bindings.map(({ name }) => implicitStringLiteral(name)))
: null
});
}
function objectRecursiveStatic(bindings) {
return tmplScopeWrapper({
bindings: bindings.map(({ name, expression }) => {
return tmplRecursiveBinding({
name: name,
expression: expression
});
}),
object: objectLiteral(bindings.map(({ name }) => {
assert(typeof name === "string");
return [ name, types.identifier(name) ];
}))
});
}
function objectRecursiveDynamic(bindings) {
throw new Error(`UNIMPLEMENTED: Dynamic bindings are not supported yet`);
return tmplDynamicScopeWrapper({
});
}
function implicitStringLiteral(node) {
if (typeof node === "string") {
return types.stringLiteral(node);
} else {
return node;
}
}
function assertKeys(keys) {
return tmplAssertKeys({ keyList: types.arrayExpression(keys) });
}
module.exports = {
name: "attribute-sets",
visitors: {
@ -55,59 +129,45 @@ module.exports = {
});
},
NixAttributeSet: (node, { defer, setContext, getContext }) => {
if (node.recursive) {
setContext(null, "attributeSets_inRecursiveSet", true);
}
// if (node.recursive) {
// setContext(null, "attributeSets_inRecursiveSet", true);
// }
return defer((node) => {
let isRecursive = node.recursive;
let [ dynamicNodes, staticNodes ] = splitFilter(node.bind, (binding) => isDynamicBinding(binding));
// let [ dynamicNodes, staticNodes ] = splitFilter(node.bind, (binding) => isDynamicBinding(binding));
// let staticBindings = staticNodes.map((binding) => {
// assert(binding.attrpath.attr.length === 1); // Nested attributes should have been desugared by this point
let staticBindings = staticNodes.map((binding) => {
// return {
// name: binding.attrpath.attr[0].name,
// expression: lazyWrapper(binding.expression)
// };
// });
let bindings = node.bind.map((binding) => {
assert(binding.attrpath.attr.length === 1); // Nested attributes should have been desugared by this point
return {
name: binding.attrpath.attr[0].name,
name: (isDynamicBinding(binding))
? binding.attrpath.attr[0]
: binding.attrpath.attr[0].name,
expression: lazyWrapper(binding.expression)
};
});
if (dynamicNodes.length > 0) {
printAst(node);
throw new Error(`UNIMPLEMENTED: Dynamic bindings are not supported yet`);
} else if (isRecursive) {
return unpackExpression(tmplScopeWrapper({
bindings: staticBindings.map(({ name, expression }) => {
return tmplRecursiveBinding({
name: name,
expression: expression
});
}),
object: types.objectExpression(staticBindings.map(({ name }) => {
return types.objectProperty(
types.stringLiteral(name),
types.identifier(name)
);
}))
}));
} else {
let object = types.objectExpression(staticBindings.map(({ name, expression }) => {
return types.objectProperty(
types.stringLiteral(name),
// TODO: Remove the isRecursive distinction here once heap-of-vars works
expression
);
}));
// Only do prototypal inheritance for `this` if we're dealing with nested recursive attribute sets
if (isRecursive && getContext("attributeSets_inRecursiveSet") === true) {
return unpackExpression(tmplExtend({
object: object
}));
let hasDynamicBindings = bindings.some((binding) => typeof binding.name !== "string");
if (isRecursive) {
if (hasDynamicBindings) {
return objectRecursiveDynamic(bindings);
} else {
return object;
return objectRecursiveStatic(bindings);
}
} else {
return objectNormal(bindings);
}
});
},

13
src/transformers/desugar-attrsets.js

@ -35,6 +35,13 @@ function unpackBindingValue(binding) {
}
}
/* FIXME: Cases to test/verify:
{ a = 4; a.b = 5; }
{ a = {}; a.b = 5; }
{ a.c = 4; a.b = 5; }
{ a = 4; a = 5; }
*/
module.exports = {
name: "desugar-attrsets",
visitors: {
@ -73,6 +80,8 @@ module.exports = {
}
} else {
// FIXME: Needs runtime check, need to *always* construct objects at runtime when dynamic bindings are involved
// FIXME: Still just desugar here? And expect another transformer to deal with the runtime checks
// desugar but do not merge, dynamic bindings cannot be combined with attribute paths on the same key
dynamicBindings.push(binding);
}
} else {
@ -80,13 +89,13 @@ module.exports = {
}
}
let staticBindingList = Object.entries(newStaticBindings).map(([ key, value ]) => {
let staticBindingList = Object.entries(newStaticBindings).map(([ _key, value ]) => {
return value;
});
// We only check this here because there are multiple things that could cause a need for desugaring, and this is simpler for now than having a separate 'check first' implementation
if (neededDesugaring) {
return NixAttributeSet([ ... staticBindingList, ... dynamicBindings ]);
return NixAttributeSet([ ... staticBindingList, ... dynamicBindings ], node.recursive);
} else {
return NoChange;
}

5
src/transformers/desugar-inherits.js

@ -18,7 +18,7 @@ module.exports = {
let tempCounter = 0;
let inherits_ = inherits.map((inherit) => {
return {
tempName: `$$jsNix$temp$${tempCounter++}`,
tempName: `$temp$${tempCounter++}`,
sourceExpression: inherit.expression,
names: inherit.attrs.attr.map((attribute) => {
assert(attribute.type === "NixAttributeIdentifier");
@ -61,7 +61,8 @@ module.exports = {
);
} else {
// ... but when recursive, we *must* specify the temporary variables as part of the recursive attribute set itself, because otherwise we don't have the ability to bidirectionally reference between fixed attributes and inherited ones; see, for example, eval-okay-scope-7.nix.
// FIXME: Mark the temporary variables as 'internal' so that they don't get returned in the final object; this would make the generated code more readable.
return {
... node,
bind: [

27
src/transformers/desugar-interpolation-expressions.js

@ -0,0 +1,27 @@
"use strict";
const NoChange = require("../astformer/actions/no-change");
module.exports = {
name: "desugar-interpolation-expressions",
visitors: {
NixAttributePath: (node) => {
if (node.attr.length === 1 && node.attr[0].type === "NixInterpolationExpression") {
// Special case: something like `{ ${foo} = "bar"; }` where the interpolation is expressed without surrounding string literal syntax
return {
... node,
// TODO: Move construction to _nixTypes
attr: [
{ type: "NixStringLiteral", parts: [
{ type: "NixInterpolationLiteral", value: "" },
node.attr[0],
{ type: "NixInterpolationLiteral", value: "" },
] },
]
};
} else {
return NoChange;
}
}
}
};

4
src/transformers/functions.js

@ -23,7 +23,7 @@ let tmplFunctionDefinitionWithFormals = template(`(
)`);
let tmplHandleArgument = template(`
const %%name%% = $$jsNix$handleArgument(%%nameString%%, %%universal%%, %%defaultArg%%);
const %%name%% = $handleArgument(%%nameString%%, %%universal%%, %%defaultArg%%);
`);
let tmplFunctionCall = template(`
@ -37,7 +37,7 @@ function typesUndefined() {
function functionDefinition({ universal, formals, body }) {
if (formals != null) {
let defaultedUniversal = universal ?? "$$jsNix$tempArg";
let defaultedUniversal = universal ?? "$tempArg";
return tmplFunctionDefinitionWithFormals({
universal: defaultedUniversal,

3
src/transformers/index.js

@ -17,7 +17,7 @@ const printAst = require("../print-ast");
// FIXME: Auto-generate argument list based on exposed API surface?
let tmplModule = template(`
module.exports = function({ builtins, $$jsNix$memoize, $$jsNix$handleArgument }) {
module.exports = function({ builtins, $memoize, $handleArgument, $assertUniqueKeys }) {
return %%contents%%;
};
`);
@ -94,6 +94,7 @@ module.exports = [
require("./desugar-inherits"),
require("./desugar-attrsets"),
require("./desugar-let-attribute-set"),
require("./desugar-interpolation-expressions"),
require("./mangle-identifiers"),
require("./let-in"),
require("./literals"),

12
src/transformers/literals.js

@ -1,18 +1,26 @@
"use strict";
const types = require("@babel/types");
const splitFilter = require("split-filter");
module.exports = {
name: "literals",
visitors: {
NixIntegerLiteral: (node) => types.numericLiteral(parseInt(node.value)),
NixFloatLiteral: (node) => types.numericLiteral(parseFloat(node.value)),
NixStringLiteral: (node) => {
NixStringLiteral: (node, { defer }) => {
if (node.parts.length === 1 && node.parts[0].type === "NixInterpolationLiteral") {
// Fast case; just a simple, non-interpolated string literal
return types.stringLiteral(node.parts[0].value);
} else {
throw new Error(`Unimplemented: string interpolation is not supported yet`);
return defer((node) => {
let [ literals, expressions ] = splitFilter(node.parts, (part) => part.type === "NixInterpolationLiteral");
return types.templateLiteral(
literals.map((node) => types.templateElement({ raw: node.value })),
expressions.map((node) => node.expression)
);
});
}
}
}

2
src/transformers/templates/lazy-wrapper.js

@ -5,7 +5,7 @@ const template = require("@babel/template").default;
const unpackExpression = require("../util/unpack-expression");
let tmplLazyWrapper = template(`(
$$jsNix$memoize(() => %%expression%%)
$memoize(() => %%expression%%)
)`);
module.exports = function lazyWrapper(expression) {

15
src/transformers/templates/object-literal.js

@ -0,0 +1,15 @@
"use strict";
const types = require("@babel/types");
module.exports = function objectLiteral(entries) {
return types.objectExpression(
entries.map(([ key, value ]) => {
if (typeof key === "string") {
return types.objectProperty(types.stringLiteral(key), value, false);
} else {
return types.objectProperty(key, value, true);
}
})
);
};

4
src/transformers/util/nix-types.js

@ -71,11 +71,11 @@ let types = module.exports = {
types.NixAttributeSet([
... bindings,
types.NixBinding(
[ types.NixAttributeIdentifier("$$jsNix$letBody") ],
[ types.NixAttributeIdentifier("$letBody") ],
body
)
], true),
[ types.NixAttributeIdentifier("$$jsNix$letBody") ]
[ types.NixAttributeIdentifier("$letBody") ]
);
}
};

14
src/transformers/util/template-expression.js

@ -0,0 +1,14 @@
"use strict";
const unpackExpression = require("./unpack-expression");
const template = require("@babel/template").default;
// Like `@babel/template`, but eliminates the wrapping ExpressionStatement
module.exports = function templateExpression(... args) {
let tmpl = template(... args);
return function (... callArgs) {
return unpackExpression(tmpl(... callArgs));
};
};
Loading…
Cancel
Save