joepie91
/
pegjs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Position tracking: Kill the |trackLineAndColumn| option

Browse Source 
Getting rid of the |trackLineAndColumn| simplifies the code generator
(by unifying two paths in the code).

The |line| and |column| functions currently always compute all the
position info from scratch, which is horribly ineffective. This will be
improved in later commit(s).
redux
David Majda 12 years ago
parent da8c455640
commit 3333cdd18d
9 changed files with 156 additions and 249 deletions
Show Stats Download Patch File Download Diff File

25
README.md
Unescape Escape View File

@ -75,8 +75,6 @@ You can tweak the generated parser with several options:
* `--cache` — makes the parser cache results, avoiding exponential parsing
time in pathological cases but making the parser slower
* `--track-line-and-column` — makes the parser track line and column
(available as `line` and `column` variables in the actions and predicates)
* `--allowed-start-rules` — comma-separated list of rules the parser will be
allowed to start parsing from (default: the first rule in the grammar)
@ -105,9 +103,6 @@ object to `PEG.buildParser`. The following options are supported:
* `cache` — if `true`, makes the parser cache results, avoiding exponential
parsing time in pathological cases but making the parser slower (default:
`false`)
* `trackLineAndColumn` — if `true`, makes the parser track line and column
(available as `line` and `column` variables in the actions and predicates)
(default: `false`)
* `allowedStartRules` — rules the parser will be allowed to start parsing from
(default: the first rule in the grammar)
* `output` — if set to `"parser"`, the method will return generated parser
@ -289,10 +284,8 @@ the initializer at the beginning of the grammar.
The code inside the predicate can also access the current parse position using
the `offset` function. It returns a zero-based character index into the input
string. If the `trackLineAndColumn` option was set to `true` when the parser was
generated (or `--track-line-and-column` was used on the command line), the code
can also access the current line and column using the `line` and `column`
functions. Both return one-based indexes.
string. The code can also access the current line and column using the `line`
and `column` functions. Both return one-based indexes.
The code inside the predicate can also access options passed to the parser using
the `options` variable.
@ -313,10 +306,8 @@ the initializer at the beginning of the grammar.
The code inside the predicate can also access the current parse position using
the `offset` function. It returns a zero-based character index into the input
string. If the `trackLineAndColumn` option was set to `true` when the parser was
generated (or `--track-line-and-column` was used on the command line), the code
can also access the current line and column using the `line` and `column`
functions. Both return one-based indexes.
string. The code can also access the current line and column using the `line`
and `column` functions. Both return one-based indexes.
The code inside the predicate can also access options passed to the parser using
the `options` variable.
@ -352,11 +343,9 @@ must be balanced.
The code inside the action can also access the parse position at the beginning
of the action's expression using the `offset` function. It returns a zero-based
character index into the input string. If the `trackLineAndColumn` option was
set to `true` when the parser was generated (or `--track-line-and-column` was
used on the command line), the code can also access the line and column at the
beginning of the action's expression using the `line` and `column` functions.
Both return one-based indexes.
character index into the input string. The code can also access the line and
column at the beginning of the action's expression using the `line` and `column`
functions. Both return one-based indexes.
The code inside the action can also access options passed to the parser using
the `options` variable.

2
benchmark/index.css
Unescape Escape View File

@ -30,5 +30,5 @@ a, a:visited { color: #3d586c; }
background-color: #f0f0f0;
}
#options #run-count { width: 3em; }
#options #cache, #options #track-line-and-column { margin-left: 2em; }
#options #cache { margin-left: 2em; }
#options #run { width: 5em; margin-left: 2em; }

2
benchmark/index.html
Unescape Escape View File

@ -13,8 +13,6 @@
<input type="text" id="run-count" value="10"> times
<input type="checkbox" id="cache">
<label for="cache">Use results cache</label>
<input type="checkbox" id="track-line-and-column">
<label for="track-line-and-column">Track line and column</label>
<input type="button" id="run" value="Run">
</div>

5
benchmark/index.js
Unescape Escape View File

@ -64,7 +64,6 @@ $("#run").click(function() {
var options = {
cache: $("#cache").is(":checked"),
trackLineAndColumn: $("#track-line-and-column").is(":checked")
};
Runner.run(benchmarks, runCount, options, {
@ -106,7 +105,7 @@ $("#run").click(function() {
},
start: function() {
$("#run-count, #cache, #track-line-and-column, #run").attr("disabled", "disabled");
$("#run-count, #cache, #run").attr("disabled", "disabled");
resultsTable.show();
$("#results-table tr").slice(1).remove();
@ -123,7 +122,7 @@ $("#run").click(function() {
$.scrollTo("max", { axis: "y", duration: 500 });
$("#run-count, #cache, #track-line-and-column, #run").removeAttr("disabled");
$("#run-count, #cache, #run").removeAttr("disabled");
}
});

7
benchmark/run
Unescape Escape View File

@ -81,7 +81,6 @@ function printHelp() {
util.puts("Options:");
util.puts(" -n, --run-count <n> number of runs (default: 10)");
util.puts(" --cache make tested parsers cache results");
util.puts(" --track-line-and-column make tested parsers track line and column");
}
function exitSuccess() {
@ -112,7 +111,7 @@ function nextArg() {
/* Main */
var runCount = 10;
var options = { trackLineAndColumn: false };
var options = { };
while (args.length > 0 && isOption(args[0])) {
switch (args[0]) {
@ -132,10 +131,6 @@ while (args.length > 0 && isOption(args[0])) {
options.cache = true;
break;
case "--track-line-and-column":
options.trackLineAndColumn = true;
break;
case "-h":
case "--help":
printHelp();

6
bin/pegjs
Unescape Escape View File

@ -25,7 +25,6 @@ function printHelp() {
util.puts(" object will be stored (default:");
util.puts(" \"module.exports\")");
util.puts(" --cache make generated parser cache results");
util.puts(" --track-line-and-column make generated parser track line and column");
util.puts(" --allowed-start-rules <rules> comma-separated list of rules the generated");
util.puts(" parser will be allowed to start parsing");
util.puts(" from (default: the first rule in the");
@ -73,7 +72,6 @@ function readStream(inputStream, callback) {
var exportVar = "module.exports";
var options = {
cache: false,
trackLineAndColumn: false,
output: "source"
};
@ -92,10 +90,6 @@ while (args.length > 0 && isOption(args[0])) {
options.cache = true;
break;
case "--track-line-and-column":
options.trackLineAndColumn = true;
break;
case "--allowed-start-rules":
nextArg();
if (args.length === 0) {

151
lib/compiler/passes/generate-code.js
Unescape Escape View File

@ -5,7 +5,6 @@ module.exports = function(ast, options) {
options = utils.clone(options);
utils.defaults(options, {
cache: false,
trackLineAndColumn: false,
allowedStartRules: [ast.startRule]
});
@ -333,10 +332,10 @@ module.exports = function(ast, options) {
' startRule = #{string(options.allowedStartRules[0])};',
' }',
' ',
' #{posInit("pos")};',
' #{posInit("reportedPos")};',
' var pos = 0;',
' var reportedPos = 0;',
' var reportFailures = 0;', // 0 = report, anything > 0 = do not report
' #{posInit("rightmostFailuresPos")};',
' var rightmostFailuresPos = 0;',
' var rightmostFailuresExpected = [];',
' #if options.cache',
' var cache = {};',
@ -372,56 +371,24 @@ module.exports = function(ast, options) {
' }',
' ',
' function offset() {',
' return #{posOffset("reportedPos")};',
' return reportedPos;',
' }',
' ',
' #if options.trackLineAndColumn',
' function line() {',
' return reportedPos.line;',
' return computePosDetails(reportedPos).line;',
' }',
' ',
' function column() {',
' return reportedPos.column;',
' return computePosDetails(reportedPos).column;',
' }',
' ',
' function clone(object) {',
' var result = {};',
' for (var key in object) {',
' result[key] = object[key];',
' }',
' return result;',
' }',
' ',
' function advance(pos, n) {',
' var endOffset = pos.offset + n;',
' ',
' for (var offset = pos.offset; offset < endOffset; offset++) {',
' var ch = input.charAt(offset);',
' if (ch === "\\n") {',
' if (!pos.seenCR) { pos.line++; }',
' pos.column = 1;',
' pos.seenCR = false;',
' } else if (ch === "\\r" || ch === "\\u2028" || ch === "\\u2029") {',
' pos.line++;',
' pos.column = 1;',
' pos.seenCR = true;',
' } else {',
' pos.column++;',
' pos.seenCR = false;',
' }',
' }',
' ',
' pos.offset += n;',
' }',
' ',
' #end',
' function matchFailed(failure) {',
' if (#{posOffset("pos")} < #{posOffset("rightmostFailuresPos")}) {',
' if (pos < rightmostFailuresPos) {',
' return;',
' }',
' ',
' if (#{posOffset("pos")} > #{posOffset("rightmostFailuresPos")}) {',
' rightmostFailuresPos = #{posClone("pos")};',
' if (pos > rightmostFailuresPos) {',
' rightmostFailuresPos = pos;',
' rightmostFailuresExpected = [];',
' }',
' ',
@ -447,8 +414,7 @@ module.exports = function(ast, options) {
' return cleanExpected;',
' }',
' ',
' #if !options.trackLineAndColumn',
' function computeErrorPosition() {',
' function computePosDetails(pos) {',
' /*',
' * The first idea was to use |String.split| to break the input up to the',
' * error position along newlines and derive the line and column from',
@ -460,7 +426,7 @@ module.exports = function(ast, options) {
' var column = 1;',
' var seenCR = false;',
' ',
' for (var i = 0; i < Math.max(pos, rightmostFailuresPos); i++) {',
' for (var i = 0; i < pos; i++) {',
' var ch = input.charAt(i);',
' if (ch === "\\n") {',
' if (!seenCR) { line++; }',
@ -478,7 +444,6 @@ module.exports = function(ast, options) {
' ',
' return { line: line, column: column };',
' }',
' #end',
' ',
' #if node.initializer',
' #block emit(node.initializer)',
@ -492,32 +457,28 @@ module.exports = function(ast, options) {
' * 1. The parser successfully parsed the whole input.',
' *',
' * - |result !== null|',
' * - |#{posOffset("pos")} === input.length|',
' * - |pos === input.length|',
' * - |rightmostFailuresExpected| may or may not contain something',
' *',
' * 2. The parser successfully parsed only a part of the input.',
' *',
' * - |result !== null|',
' * - |#{posOffset("pos")} < input.length|',
' * - |pos < input.length|',
' * - |rightmostFailuresExpected| may or may not contain something',
' *',
' * 3. The parser did not successfully parse any part of the input.',
' *',
' * - |result === null|',
' * - |#{posOffset("pos")} === 0|',
' * - |pos === 0|',
' * - |rightmostFailuresExpected| contains at least one failure',
' *',
' * All code following this comment (including called functions) must',
' * handle these states.',
' */',
' if (result === null || #{posOffset("pos")} !== input.length) {',
' var offset = Math.max(#{posOffset("pos")}, #{posOffset("rightmostFailuresPos")});',
' if (result === null || pos !== input.length) {',
' var offset = Math.max(pos, rightmostFailuresPos);',
' var found = offset < input.length ? input.charAt(offset) : null;',
' #if options.trackLineAndColumn',
' var errorPosition = #{posOffset("pos")} > #{posOffset("rightmostFailuresPos")} ? pos : rightmostFailuresPos;',
' #else',
' var errorPosition = computeErrorPosition();',
' #end',
' var errorPosition = computePosDetails(Math.max(pos, rightmostFailuresPos));',
' ',
' throw new this.SyntaxError(',
' cleanupExpected(rightmostFailuresExpected),',
@ -573,10 +534,10 @@ module.exports = function(ast, options) {
rule: [
'function parse_#{node.name}() {',
' #if options.cache',
' var cacheKey = "#{node.name}@" + #{posOffset("pos")};',
' var cacheKey = "#{node.name}@" + pos;',
' var cachedResult = cache[cacheKey];',
' if (cachedResult) {',
' pos = #{posClone("cachedResult.nextPos")};',
' pos = cachedResult.nextPos;',
' return cachedResult.result;',
' }',
' ',
@ -589,7 +550,7 @@ module.exports = function(ast, options) {
' #if options.cache',
' ',
' cache[cacheKey] = {',
' nextPos: #{posClone("pos")},',
' nextPos: pos,',
' result: #{r(node.expression.resultIndex)}',
' };',
' #end',
@ -614,18 +575,18 @@ module.exports = function(ast, options) {
'}'
],
action: [
'#{posSave(node)};',
'#{r(node.posIndex)} = pos;',
'#block emit(node.expression)',
'if (#{r(node.resultIndex)} !== null) {',
' reportedPos = #{posClone(r(node.posIndex))};',
' reportedPos = #{r(node.posIndex)};',
' #{r(node.resultIndex)} = (function(#{keys(node.params).join(", ")}) {#{node.code}})(#{map(values(node.params), r).join(", ")});',
'}',
'if (#{r(node.resultIndex)} === null) {',
' #{posRestore(node)};',
' pos = #{r(node.posIndex)};',
'}'
],
sequence: [
'#{posSave(node)};',
'#{r(node.posIndex)} = pos;',
'#block code'
],
"sequence.iteration": [
@ -634,26 +595,26 @@ module.exports = function(ast, options) {
' #block code',
'} else {',
' #{r(node.resultIndex)} = null;',
' #{posRestore(node)};',
' pos = #{r(node.posIndex)};',
'}'
],
"sequence.inner": [
'#{r(node.resultIndex)} = [#{map(pluck(node.elements, "resultIndex"), r).join(", ")}];'
],
simple_and: [
'#{posSave(node)};',
'#{r(node.posIndex)} = pos;',
'reportFailures++;',
'#block emit(node.expression)',
'reportFailures--;',
'if (#{r(node.resultIndex)} !== null) {',
' #{r(node.resultIndex)} = "";',
' #{posRestore(node)};',
' pos = #{r(node.posIndex)};',
'} else {',
' #{r(node.resultIndex)} = null;',
'}'
],
simple_not: [
'#{posSave(node)};',
'#{r(node.posIndex)} = pos;',
'reportFailures++;',
'#block emit(node.expression)',
'reportFailures--;',
@ -661,15 +622,15 @@ module.exports = function(ast, options) {
' #{r(node.resultIndex)} = "";',
'} else {',
' #{r(node.resultIndex)} = null;',
' #{posRestore(node)};',
' pos = #{r(node.posIndex)};',
'}'
],
semantic_and: [
'reportedPos = #{posClone("pos")};',
'reportedPos = pos;',
'#{r(node.resultIndex)} = (function(#{keys(node.params).join(", ")}) {#{node.code}})(#{map(values(node.params), r).join(", ")}) ? "" : null;'
],
semantic_not: [
'reportedPos = #{posClone("pos")};',
'reportedPos = pos;',
'#{r(node.resultIndex)} = (function(#{keys(node.params).join(", ")}) {#{node.code}})(#{map(values(node.params), r).join(", ")}) ? null : "";'
],
optional: [
@ -705,9 +666,9 @@ module.exports = function(ast, options) {
'#else',
' #if !node.ignoreCase',
' #if node.value.length === 1',
' if (input.charCodeAt(#{posOffset("pos")}) === #{node.value.charCodeAt(0)}) {',
' if (input.charCodeAt(pos) === #{node.value.charCodeAt(0)}) {',
' #else',
' if (input.substr(#{posOffset("pos")}, #{node.value.length}) === #{string(node.value)}) {',
' if (input.substr(pos, #{node.value.length}) === #{string(node.value)}) {',
' #end',
' #else',
/*
@ -718,14 +679,14 @@ module.exports = function(ast, options) {
* meaning the result of lowercasing a character can be more
* characters.
*/
' if (input.substr(#{posOffset("pos")}, #{node.value.length}).toLowerCase() === #{string(node.value.toLowerCase())}) {',
' if (input.substr(pos, #{node.value.length}).toLowerCase() === #{string(node.value.toLowerCase())}) {',
' #end',
' #if !node.ignoreCase',
' #{r(node.resultIndex)} = #{string(node.value)};',
' #else',
' #{r(node.resultIndex)} = input.substr(#{posOffset("pos")}, #{node.value.length});',
' #{r(node.resultIndex)} = input.substr(pos, #{node.value.length});',
' #end',
' #{posAdvance(node.value.length)};',
' #{node.value.length > 1 ? "pos += " + node.value.length : "pos++"};',
' } else {',
' #{r(node.resultIndex)} = null;',
' if (reportFailures === 0) {',
@ -735,9 +696,9 @@ module.exports = function(ast, options) {
'#end'
],
"class": [
'if (#{regexp}.test(input.charAt(#{posOffset("pos")}))) {',
' #{r(node.resultIndex)} = input.charAt(#{posOffset("pos")});',
' #{posAdvance(1)};',
'if (#{regexp}.test(input.charAt(pos))) {',
' #{r(node.resultIndex)} = input.charAt(pos);',
' pos++;',
'} else {',
' #{r(node.resultIndex)} = null;',
' if (reportFailures === 0) {',
@ -746,9 +707,9 @@ module.exports = function(ast, options) {
'}'
],
any: [
'if (input.length > #{posOffset("pos")}) {',
' #{r(node.resultIndex)} = input.charAt(#{posOffset("pos")});',
' #{posAdvance(1)};',
'if (input.length > pos) {',
' #{r(node.resultIndex)} = input.charAt(pos);',
' pos++;',
'} else {',
' #{r(node.resultIndex)} = null;',
' if (reportFailures === 0) {',
@ -777,34 +738,6 @@ module.exports = function(ast, options) {
vars.r = function(index) { return "r" + index; };
/* Position-handling macros */
if (options.trackLineAndColumn) {
vars.posInit = function(name) {
return "var "
+ name
+ " = "
+ "{ offset: 0, line: 1, column: 1, seenCR: false }";
};
vars.posClone = function(name) { return "clone(" + name + ")"; };
vars.posOffset = function(name) { return name + ".offset"; };
vars.posAdvance = function(n) { return "advance(pos, " + n + ")"; };
} else {
vars.posInit = function(name) { return "var " + name + " = 0"; };
vars.posClone = function(name) { return name; };
vars.posOffset = function(name) { return name; };
vars.posAdvance = function(n) {
return n === 1 ? "pos++" : "pos += " + n;
};
}
vars.posSave = function(node) {
return vars.r(node.posIndex) + " = " + vars.posClone("pos");
};
vars.posRestore = function(node) {
return "pos" + " = " + vars.posClone(vars.r(node.posIndex));
};
return templates[name](vars);
}

14
lib/parser.js
Unescape Escape View File

@ -96,6 +96,14 @@ module.exports = (function(){
return reportedPos;
}
function line() {
return computePosDetails(reportedPos).line;
}
function column() {
return computePosDetails(reportedPos).column;
}
function matchFailed(failure) {
if (pos < rightmostFailuresPos) {
return;
@ -2817,7 +2825,7 @@ module.exports = (function(){
return cleanExpected;
}
function computeErrorPosition() {
function computePosDetails(pos) {
/*
* The first idea was to use |String.split| to break the input up to the
* error position along newlines and derive the line and column from
@ -2829,7 +2837,7 @@ module.exports = (function(){
var column = 1;
var seenCR = false;
for (var i = 0; i < Math.max(pos, rightmostFailuresPos); i++) {
for (var i = 0; i < pos; i++) {
var ch = input.charAt(i);
if (ch === "\n") {
if (!seenCR) { line++; }
@ -2881,7 +2889,7 @@ module.exports = (function(){
if (result === null || pos !== input.length) {
var offset = Math.max(pos, rightmostFailuresPos);
var found = offset < input.length ? input.charAt(offset) : null;
var errorPosition = computeErrorPosition();
var errorPosition = computePosDetails(Math.max(pos, rightmostFailuresPos));
throw new this.SyntaxError(
cleanupExpected(rightmostFailuresExpected),

13
spec/generated-parser.spec.js
Unescape Escape View File

@ -1,9 +1,6 @@
describe("generated parser", function() {
function vary(names, block) {
var values = {
trackLineAndColumn: [false, true],
cache: [false, true]
};
var values = { cache: [false, true] };
function varyStep(names, options) {
var clonedOptions = {}, key, name, i;
@ -35,7 +32,7 @@ describe("generated parser", function() {
}
function varyAll(block) {
vary(["cache", "trackLineAndColumn"], block);
vary(["cache"], block);
}
beforeEach(function() {
@ -267,7 +264,6 @@ describe("generated parser", function() {
expect(parser).toParse("ab", ["a", 1]);
});
if (options.trackLineAndColumn) {
it("can use the |line| and |column| functions to get the current line and column", function() {
var parser = PEG.buildParser([
'{ var result; }',
@ -290,7 +286,6 @@ describe("generated parser", function() {
expect(parser).toParse("1\u2028x", [2, 1]); // line separator
expect(parser).toParse("1\u2029x", [2, 1]); // paragraph separator
});
}
it("can use variables defined in the initializer", function() {
var parser = PEG.buildParser([
@ -431,7 +426,6 @@ describe("generated parser", function() {
expect(parser).toParse("a", ["a", ""]);
});
if (options.trackLineAndColumn) {
it("can use the |line| and |column| functions to get the current line and column", function() {
var parser = PEG.buildParser([
'{ var result; }',
@ -454,7 +448,6 @@ describe("generated parser", function() {
expect(parser).toParse("1\u2028x", [2, 1]); // line separator
expect(parser).toParse("1\u2029x", [2, 1]); // paragraph separator
});
}
it("can use variables defined in the initializer", function() {
var parser = PEG.buildParser([
@ -515,7 +508,6 @@ describe("generated parser", function() {
expect(parser).toParse("a", ["a", ""]);
});
if (options.trackLineAndColumn) {
it("can use the |line| and |column| functions to get the current line and column", function() {
var parser = PEG.buildParser([
'{ var result; }',
@ -538,7 +530,6 @@ describe("generated parser", function() {
expect(parser).toParse("1\u2028x", [2, 1]); // line separator
expect(parser).toParse("1\u2029x", [2, 1]); // paragraph separator
});
}
it("can use variables defined in the initializer", function() {
var parser = PEG.buildParser([

Write Preview
Loading…
Cancel
Save