From 3772914db5a7c16c57de125b6d314a98391885fe Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 24 Apr 2017 19:46:37 -0400 Subject: [PATCH 1/6] Implement Export{Default,Namespace}Specifier parsing ourselves. Related: https://github.com/ternjs/acorn/pull/541 --- lib/parsers/acorn-extensions.js | 191 ++++++++++++++++++++++++++++++++ lib/parsers/acorn.js | 11 +- lib/parsers/top-level.js | 12 +- test/export-tests.js | 5 +- 4 files changed, 196 insertions(+), 23 deletions(-) create mode 100644 lib/parsers/acorn-extensions.js diff --git a/lib/parsers/acorn-extensions.js b/lib/parsers/acorn-extensions.js new file mode 100644 index 00000000..55ce409d --- /dev/null +++ b/lib/parsers/acorn-extensions.js @@ -0,0 +1,191 @@ +"use strict"; + +const tt = require("acorn").tokTypes; +const hasOwn = Object.prototype.hasOwnProperty; + +exports.fix = function (parser) { + // It's not Reify's job to enforce strictness. + parser.strict = false; + + // Tolerate recoverable parse errors. + parser.raiseRecoverable = noopRaiseRecoverable; + + // Lookahead method. + parser.lookAhead = lookAhead; + + // Export-related modifications. + parser.parseExport = parseExport; + parser.isExportDefaultSpecifier = isExportDefaultSpecifier; + parser.parseExportSpecifiersMaybe = parseExportSpecifiersMaybe; + parser.parseExportFromWithCheck = parseExportFromWithCheck; + parser.parseExportFrom = parseExportFrom; + parser.checkExport = checkExport; + parser.shouldParseExportDeclaration = shouldParseExportDeclaration; +}; + +function noopRaiseRecoverable() {} + +function parseExport(node, exports) { + this.next(); + if (this.type === tt.star) { + const specifier = this.startNode(); + this.next(); + if (this.eatContextual("as")) { + // export * as ns from '...' + specifier.exported = this.parseIdent(true); + node.specifiers = [ + this.finishNode(specifier, "ExportNamespaceSpecifier") + ]; + this.parseExportSpecifiersMaybe(node); + this.parseExportFromWithCheck(node, exports, true); + } else { + // export * from '...' + this.parseExportFromWithCheck(node, exports, true); + return this.finishNode(node, "ExportAllDeclaration"); + } + } else if (this.isExportDefaultSpecifier()) { + // export def from '...' + const specifier = this.startNode(); + specifier.exported = this.parseIdent(true); + node.specifiers = [ + this.finishNode(specifier, "ExportDefaultSpecifier") + ]; + if (this.type === tt.comma && + this.lookAhead(1).type === tt.star) { + // export def, * as ns from '...' + this.expect(tt.comma); + const specifier = this.startNode(); + this.expect(tt.star); + this.expectContextual("as"); + specifier.exported = this.parseIdent(true); + node.specifiers.push( + this.finishNode(specifier, "ExportNamespaceSpecifier") + ); + } else { + // export def, { x, y as z } from '...' + this.parseExportSpecifiersMaybe(node); + } + this.parseExportFromWithCheck(node, exports, true); + } else if (this.eat(tt._default)) { + // export default ... + this.checkExport(exports, "default", this.lastTokStart); + let isAsync; + if (this.type === tt._function || (isAsync = this.isAsyncFunction())) { + let fNode = this.startNode(); + this.next(); + if (isAsync) this.next(); + node.declaration = this.parseFunction(fNode, "nullableID", false, isAsync); + } else if (this.type === tt._class) { + let cNode = this.startNode(); + node.declaration = this.parseClass(cNode, "nullableID"); + } else { + node.declaration = this.parseMaybeAssign(); + this.semicolon(); + } + return this.finishNode(node, "ExportDefaultDeclaration"); + } else if (this.shouldParseExportDeclaration()) { + // export var|const|let|function|class ... + node.declaration = this.parseStatement(true); + if (node.declaration.type === "VariableDeclaration") { + this.checkVariableExport(exports, node.declaration.declarations); + } else { + this.checkExport( + exports, + node.declaration.id.name, + node.declaration.id.start + ); + } + node.specifiers = []; + node.source = null; + } else { + // export { x, y as z } [from '...'] + node.declaration = null; + node.specifiers = this.parseExportSpecifiers(exports); + this.parseExportFrom(node, false); + } + return this.finishNode(node, "ExportNamedDeclaration"); +} + +function lookAhead(n) { + const old = Object.assign(Object.create(null), this); + while (n-- > 0) this.nextToken(); + const copy = Object.assign(Object.create(null), this); + Object.assign(this, old); + return copy; +} + +function isExportDefaultSpecifier() { + if (this.type !== tt.name) { + return false; + } + + const lookAhead = this.lookAhead(1); + return lookAhead.type === tt.comma || + (lookAhead.type === tt.name && + lookAhead.value === "from"); +} + +function parseExportSpecifiersMaybe(node) { + if (this.eat(tt.comma)) { + node.specifiers.push.apply( + node.specifiers, + this.parseExportSpecifiers() + ); + } +} + +function parseExportFromWithCheck(node, exports, expect) { + this.parseExportFrom(node, expect); + + if (node.specifiers) { + for (let i = 0; i < node.specifiers.length; i++) { + const s = node.specifiers[i]; + const exported = s.exported; + this.checkExport(exports, exported.name, exported.start); + } + } +} + +function parseExportFrom(node, expect) { + if (this.eatContextual("from")) { + node.source = this.type === tt.string + ? this.parseExprAtom() + : this.unexpected(); + } else { + if (node.specifiers) { + // check for keywords used as local names + for (let i = 0; i < node.specifiers.length; i++) { + const local = node.specifiers[i].local; + if (local && (this.keywords.test(local.name) || + this.reservedWords.test(local.name))) { + this.unexpected(local.start); + } + } + } + + if (expect) { + this.unexpected(); + } else { + node.source = null; + } + } + + this.semicolon(); +} + +function checkExport(exports, name, pos) { + if (!exports) return; + if (hasOwn.call(exports, name)) { + this.raiseRecoverable(pos, "Duplicate export '" + name + "'"); + } + exports[name] = true; +} + +function shouldParseExportDeclaration() { + return this.type.keyword === "var" || + this.type.keyword === "const" || + this.type.keyword === "class" || + this.type.keyword === "function" || + this.isLet() || + this.isAsyncFunction(); +} diff --git a/lib/parsers/acorn.js b/lib/parsers/acorn.js index 97948a1a..65f6f723 100644 --- a/lib/parsers/acorn.js +++ b/lib/parsers/acorn.js @@ -1,6 +1,7 @@ "use strict"; const acorn = require("acorn"); +const fixParser = require("./acorn-extensions.js").fix; exports.options = { ecmaVersion: 8, @@ -12,16 +13,8 @@ exports.options = { function acornParse(code) { const parser = new acorn.Parser(exports.options, code); - - // It's not Reify's job to enforce strictness. - parser.strict = false; - - // Tolerate recoverable parse errors. - parser.raiseRecoverable = noopRaiseRecoverable; - + fixParser(parser); return parser.parse(); } -function noopRaiseRecoverable() {} - exports.parse = acornParse; diff --git a/lib/parsers/top-level.js b/lib/parsers/top-level.js index 17fd775f..03f1c678 100644 --- a/lib/parsers/top-level.js +++ b/lib/parsers/top-level.js @@ -1,6 +1,7 @@ "use strict"; const acorn = require("acorn"); +const fixParser = require("./acorn-extensions.js").fix; exports.options = { ecmaVersion: 8, @@ -26,19 +27,10 @@ function quickParseBlock() { function topLevelParse(code) { const parser = new acorn.Parser(exports.options, code); - + fixParser(parser); // Override the Parser's parseBlock method. parser.parseBlock = quickParseBlock; - - // It's not Reify's job to enforce strictness. - parser.strict = false; - - // Tolerate recoverable parse errors. - parser.raiseRecoverable = noopRaiseRecoverable; - return parser.parse(); } -function noopRaiseRecoverable() {} - exports.parse = topLevelParse; diff --git a/test/export-tests.js b/test/export-tests.js index 5175d4b6..ce7f68a2 100644 --- a/test/export-tests.js +++ b/test/export-tests.js @@ -1,6 +1,4 @@ const assert = require("assert"); -const parserSupportsExportFromExtensions = - process.env.REIFY_PARSER === "babylon"; describe("export declarations", () => { it("should allow * exports", () => { @@ -206,8 +204,7 @@ describe("export declarations", () => { }); }); - (parserSupportsExportFromExtensions ? it : xit - )("should support export-from extensions", () => { + it("should support export-from extensions", () => { import { def1, def2, def3, ns1, ns2, ns3, From 49b4a18bb27038ad302f036173106dc32b2b44ee Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 25 Apr 2017 11:25:47 -0400 Subject: [PATCH 2/6] Better naming for functions exported by acorn-extensions.js. --- lib/parsers/acorn-extensions.js | 13 ++++++++++--- lib/parsers/acorn.js | 16 +++++++++++++--- lib/parsers/top-level.js | 17 ++++++++++++++--- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/lib/parsers/acorn-extensions.js b/lib/parsers/acorn-extensions.js index 55ce409d..abefc9e7 100644 --- a/lib/parsers/acorn-extensions.js +++ b/lib/parsers/acorn-extensions.js @@ -3,13 +3,22 @@ const tt = require("acorn").tokTypes; const hasOwn = Object.prototype.hasOwnProperty; -exports.fix = function (parser) { +exports.enableAll = function (parser) { + exports.enableTolerance(parser); + exports.enableExportExtensions(parser); +}; + +exports.enableTolerance = function (parser) { // It's not Reify's job to enforce strictness. parser.strict = false; // Tolerate recoverable parse errors. parser.raiseRecoverable = noopRaiseRecoverable; +}; +function noopRaiseRecoverable() {} + +exports.enableExportExtensions = function (parser) { // Lookahead method. parser.lookAhead = lookAhead; @@ -23,8 +32,6 @@ exports.fix = function (parser) { parser.shouldParseExportDeclaration = shouldParseExportDeclaration; }; -function noopRaiseRecoverable() {} - function parseExport(node, exports) { this.next(); if (this.type === tt.star) { diff --git a/lib/parsers/acorn.js b/lib/parsers/acorn.js index 65f6f723..fa468989 100644 --- a/lib/parsers/acorn.js +++ b/lib/parsers/acorn.js @@ -1,7 +1,7 @@ "use strict"; -const acorn = require("acorn"); -const fixParser = require("./acorn-extensions.js").fix; +let acorn = null; +let acornExtensions = null; exports.options = { ecmaVersion: 8, @@ -12,8 +12,18 @@ exports.options = { }; function acornParse(code) { + if (acorn === null) { + acorn = require("acorn"); + } + + if (acornExtensions === null) { + acornExtensions = require("./acorn-extensions.js"); + } + const parser = new acorn.Parser(exports.options, code); - fixParser(parser); + + acornExtensions.enableAll(parser); + return parser.parse(); } diff --git a/lib/parsers/top-level.js b/lib/parsers/top-level.js index 03f1c678..81c5621e 100644 --- a/lib/parsers/top-level.js +++ b/lib/parsers/top-level.js @@ -1,7 +1,7 @@ "use strict"; -const acorn = require("acorn"); -const fixParser = require("./acorn-extensions.js").fix; +let acorn = null; +let acornExtensions = null; exports.options = { ecmaVersion: 8, @@ -26,10 +26,21 @@ function quickParseBlock() { } function topLevelParse(code) { + if (acorn === null) { + acorn = require("acorn"); + } + + if (acornExtensions === null) { + acornExtensions = require("./acorn-extensions.js"); + } + const parser = new acorn.Parser(exports.options, code); - fixParser(parser); + + acornExtensions.enableAll(parser); + // Override the Parser's parseBlock method. parser.parseBlock = quickParseBlock; + return parser.parse(); } From 5366b4c6e2dd7bfa957d28e86257ee8a4513f88f Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 25 Apr 2017 11:29:07 -0400 Subject: [PATCH 3/6] Eliminate useless Parser#checkExport override. The only point of this method besides updating the map of exported symbol names was to complain if there were duplicate export names, but we override the raiseRecoverable method with noopRaiseRecoverable anyway, so there's really no need to override checkExport. --- lib/parsers/acorn-extensions.js | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/lib/parsers/acorn-extensions.js b/lib/parsers/acorn-extensions.js index abefc9e7..73a7f5a6 100644 --- a/lib/parsers/acorn-extensions.js +++ b/lib/parsers/acorn-extensions.js @@ -1,7 +1,6 @@ "use strict"; const tt = require("acorn").tokTypes; -const hasOwn = Object.prototype.hasOwnProperty; exports.enableAll = function (parser) { exports.enableTolerance(parser); @@ -28,7 +27,6 @@ exports.enableExportExtensions = function (parser) { parser.parseExportSpecifiersMaybe = parseExportSpecifiersMaybe; parser.parseExportFromWithCheck = parseExportFromWithCheck; parser.parseExportFrom = parseExportFrom; - parser.checkExport = checkExport; parser.shouldParseExportDeclaration = shouldParseExportDeclaration; }; @@ -75,7 +73,7 @@ function parseExport(node, exports) { this.parseExportFromWithCheck(node, exports, true); } else if (this.eat(tt._default)) { // export default ... - this.checkExport(exports, "default", this.lastTokStart); + exports.default = true; let isAsync; if (this.type === tt._function || (isAsync = this.isAsyncFunction())) { let fNode = this.startNode(); @@ -96,11 +94,7 @@ function parseExport(node, exports) { if (node.declaration.type === "VariableDeclaration") { this.checkVariableExport(exports, node.declaration.declarations); } else { - this.checkExport( - exports, - node.declaration.id.name, - node.declaration.id.start - ); + exports[node.declaration.id.name] = true; } node.specifiers = []; node.source = null; @@ -148,7 +142,7 @@ function parseExportFromWithCheck(node, exports, expect) { for (let i = 0; i < node.specifiers.length; i++) { const s = node.specifiers[i]; const exported = s.exported; - this.checkExport(exports, exported.name, exported.start); + exports[exported.name] = true; } } } @@ -180,14 +174,6 @@ function parseExportFrom(node, expect) { this.semicolon(); } -function checkExport(exports, name, pos) { - if (!exports) return; - if (hasOwn.call(exports, name)) { - this.raiseRecoverable(pos, "Duplicate export '" + name + "'"); - } - exports[name] = true; -} - function shouldParseExportDeclaration() { return this.type.keyword === "var" || this.type.keyword === "const" || From 5f7f0a1bca732b68ed6c09361b1b2d866e9c97fc Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 25 Apr 2017 11:37:40 -0400 Subject: [PATCH 4/6] Combine parseExportFromWithCheck with parseExportFrom and simplify. The Reify parser is more like the "loose" Acorn parser in that it doesn't care about enforcing expectations as long they don't affect the results of the parsing. So a loop that serves only to call this.unexpected is a loop that we can do without. --- lib/parsers/acorn-extensions.js | 39 ++++++--------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/lib/parsers/acorn-extensions.js b/lib/parsers/acorn-extensions.js index 73a7f5a6..93606c41 100644 --- a/lib/parsers/acorn-extensions.js +++ b/lib/parsers/acorn-extensions.js @@ -25,7 +25,6 @@ exports.enableExportExtensions = function (parser) { parser.parseExport = parseExport; parser.isExportDefaultSpecifier = isExportDefaultSpecifier; parser.parseExportSpecifiersMaybe = parseExportSpecifiersMaybe; - parser.parseExportFromWithCheck = parseExportFromWithCheck; parser.parseExportFrom = parseExportFrom; parser.shouldParseExportDeclaration = shouldParseExportDeclaration; }; @@ -42,10 +41,10 @@ function parseExport(node, exports) { this.finishNode(specifier, "ExportNamespaceSpecifier") ]; this.parseExportSpecifiersMaybe(node); - this.parseExportFromWithCheck(node, exports, true); + this.parseExportFrom(node, exports); } else { // export * from '...' - this.parseExportFromWithCheck(node, exports, true); + this.parseExportFrom(node, exports); return this.finishNode(node, "ExportAllDeclaration"); } } else if (this.isExportDefaultSpecifier()) { @@ -70,7 +69,7 @@ function parseExport(node, exports) { // export def, { x, y as z } from '...' this.parseExportSpecifiersMaybe(node); } - this.parseExportFromWithCheck(node, exports, true); + this.parseExportFrom(node, exports); } else if (this.eat(tt._default)) { // export default ... exports.default = true; @@ -102,7 +101,7 @@ function parseExport(node, exports) { // export { x, y as z } [from '...'] node.declaration = null; node.specifiers = this.parseExportSpecifiers(exports); - this.parseExportFrom(node, false); + this.parseExportFrom(node, exports); } return this.finishNode(node, "ExportNamedDeclaration"); } @@ -135,8 +134,9 @@ function parseExportSpecifiersMaybe(node) { } } -function parseExportFromWithCheck(node, exports, expect) { - this.parseExportFrom(node, expect); +function parseExportFrom(node, exports) { + const hasFrom = this.eatContextual("from") && this.type === tt.string; + node.source = hasFrom ? this.parseExprAtom() : null; if (node.specifiers) { for (let i = 0; i < node.specifiers.length; i++) { @@ -145,31 +145,6 @@ function parseExportFromWithCheck(node, exports, expect) { exports[exported.name] = true; } } -} - -function parseExportFrom(node, expect) { - if (this.eatContextual("from")) { - node.source = this.type === tt.string - ? this.parseExprAtom() - : this.unexpected(); - } else { - if (node.specifiers) { - // check for keywords used as local names - for (let i = 0; i < node.specifiers.length; i++) { - const local = node.specifiers[i].local; - if (local && (this.keywords.test(local.name) || - this.reservedWords.test(local.name))) { - this.unexpected(local.start); - } - } - } - - if (expect) { - this.unexpected(); - } else { - node.source = null; - } - } this.semicolon(); } From 4b6d0c3b7501a733772f97d120dd941c3fa20870 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 25 Apr 2017 11:43:26 -0400 Subject: [PATCH 5/6] Avoid making extra copy of parser state while looking ahead. --- lib/parsers/acorn-extensions.js | 38 +++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/lib/parsers/acorn-extensions.js b/lib/parsers/acorn-extensions.js index 93606c41..1563a781 100644 --- a/lib/parsers/acorn-extensions.js +++ b/lib/parsers/acorn-extensions.js @@ -18,8 +18,8 @@ exports.enableTolerance = function (parser) { function noopRaiseRecoverable() {} exports.enableExportExtensions = function (parser) { - // Lookahead method. - parser.lookAhead = lookAhead; + // Our custom lookahead method. + parser.withLookAhead = withLookAhead; // Export-related modifications. parser.parseExport = parseExport; @@ -55,7 +55,7 @@ function parseExport(node, exports) { this.finishNode(specifier, "ExportDefaultSpecifier") ]; if (this.type === tt.comma && - this.lookAhead(1).type === tt.star) { + peekNextType(this) === tt.star) { // export def, * as ns from '...' this.expect(tt.comma); const specifier = this.startNode(); @@ -106,23 +106,33 @@ function parseExport(node, exports) { return this.finishNode(node, "ExportNamedDeclaration"); } -function lookAhead(n) { +// Calls the given callback with the state of the parser temporarily +// advanced by calling this.nextToken() n times, then rolls the parser +// back to its original state and returns whatever the callback returned. +function withLookAhead(n, callback) { const old = Object.assign(Object.create(null), this); while (n-- > 0) this.nextToken(); - const copy = Object.assign(Object.create(null), this); - Object.assign(this, old); - return copy; + try { + return callback.call(this); + } finally { + Object.assign(this, old); + } +} + +function peekNextType(parser) { + return parser.withLookAhead(1, () => parser.type); } function isExportDefaultSpecifier() { - if (this.type !== tt.name) { - return false; - } + return this.type === tt.name && + this.withLookAhead(1, isCommaOrFrom); +} - const lookAhead = this.lookAhead(1); - return lookAhead.type === tt.comma || - (lookAhead.type === tt.name && - lookAhead.value === "from"); +function isCommaOrFrom() { + // Note: `this` should be the parser object. + return this.type === tt.comma || + (this.type === tt.name && + this.value === "from"); } function parseExportSpecifiersMaybe(node) { From 80b30d040a40f299839d9eccdf0516ff327d189e Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 25 Apr 2017 13:09:19 -0400 Subject: [PATCH 6/6] Pass parser as argument rather than `this` to withLookAhead callback. --- lib/parsers/acorn-extensions.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/parsers/acorn-extensions.js b/lib/parsers/acorn-extensions.js index 1563a781..952010da 100644 --- a/lib/parsers/acorn-extensions.js +++ b/lib/parsers/acorn-extensions.js @@ -113,7 +113,7 @@ function withLookAhead(n, callback) { const old = Object.assign(Object.create(null), this); while (n-- > 0) this.nextToken(); try { - return callback.call(this); + return callback(this); } finally { Object.assign(this, old); } @@ -128,11 +128,10 @@ function isExportDefaultSpecifier() { this.withLookAhead(1, isCommaOrFrom); } -function isCommaOrFrom() { - // Note: `this` should be the parser object. - return this.type === tt.comma || - (this.type === tt.name && - this.value === "from"); +function isCommaOrFrom(parser) { + return parser.type === tt.comma || + (parser.type === tt.name && + parser.value === "from"); } function parseExportSpecifiersMaybe(node) {