From 75abcb728c0b58d2f7d229ab3c80843f93c09ca2 Mon Sep 17 00:00:00 2001 From: Ganuta Aleksey Date: Sat, 14 Dec 2019 20:47:48 +0300 Subject: [PATCH 1/2] typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3fc087b..503ea12 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ yarn add --dev easygettext ##### HTML token extraction Simply invoke the tool on the templates you want to extract a POT dictionary template from. -The optional '--ouput' argument enables you to directly output to a file. +The optional '--output' argument enables you to directly output to a file. ``` gettext-extract --output dictionary.pot foo.html bar.pug component.vue sourcefile.js From 078fa45b1d807abfa936ecf71b25c188195800ec Mon Sep 17 00:00:00 2001 From: Ganuta Aleksey Date: Sat, 14 Dec 2019 20:51:31 +0300 Subject: [PATCH 2/2] feature: added babel parser support for javascript parsing. --- package-lock.json | 65 +++++++++------------------------------ package.json | 1 + src/extract-cli.js | 9 +++--- src/extract.js | 4 +-- src/javascript-extract.js | 49 +++++++++++++++++++++-------- 5 files changed, 59 insertions(+), 69 deletions(-) diff --git a/package-lock.json b/package-lock.json index b7f1300..e736b77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,16 +8,14 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz", "integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==", - "dev": true, "requires": { "@babel/highlight": "^7.0.0" } }, "@babel/core": { "version": "7.7.5", - "resolved": "https://npm.polydev.blue/@babel%2fcore/-/core-7.7.5.tgz", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.7.5.tgz", "integrity": "sha512-M42+ScN4+1S9iB6f+TL7QBpoQETxbclx+KNoKJABghnKYE+fMzSGqst0BZJc8CpI625bwPwYgUyRvxZ+0mZzpw==", - "dev": true, "requires": { "@babel/code-frame": "^7.5.5", "@babel/generator": "^7.7.4", @@ -39,22 +37,14 @@ "version": "7.5.5", "resolved": "https://npm.polydev.blue/@babel%2fcode-frame/-/code-frame-7.5.5.tgz", "integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==", - "dev": true, "requires": { "@babel/highlight": "^7.0.0" } }, - "lodash": { - "version": "4.17.15", - "resolved": "https://npm.polydev.blue/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", - "dev": true - }, "source-map": { "version": "0.5.7", "resolved": "https://npm.polydev.blue/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", - "dev": true + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" } } }, @@ -62,7 +52,6 @@ "version": "7.7.4", "resolved": "https://npm.polydev.blue/@babel%2fgenerator/-/generator-7.7.4.tgz", "integrity": "sha512-m5qo2WgdOJeyYngKImbkyQrnUN1mPceaG5BV+G0E3gWsa4l/jCSryWJdM2x8OuGAOyh+3d5pVYfZWCiNFtynxg==", - "dev": true, "requires": { "@babel/types": "^7.7.4", "jsesc": "^2.5.1", @@ -73,14 +62,12 @@ "lodash": { "version": "4.17.15", "resolved": "https://npm.polydev.blue/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", - "dev": true + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, "source-map": { "version": "0.5.7", "resolved": "https://npm.polydev.blue/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", - "dev": true + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" } } }, @@ -88,7 +75,6 @@ "version": "7.7.4", "resolved": "https://npm.polydev.blue/@babel%2fhelper-function-name/-/helper-function-name-7.7.4.tgz", "integrity": "sha512-AnkGIdiBhEuiwdoMnKm7jfPfqItZhgRaZfMg1XX3bS25INOnLPjPG1Ppnajh8eqgt5kPJnfqrRHqFqmjKDZLzQ==", - "dev": true, "requires": { "@babel/helper-get-function-arity": "^7.7.4", "@babel/template": "^7.7.4", @@ -99,7 +85,6 @@ "version": "7.7.4", "resolved": "https://npm.polydev.blue/@babel%2fhelper-get-function-arity/-/helper-get-function-arity-7.7.4.tgz", "integrity": "sha512-QTGKEdCkjgzgfJ3bAyRwF4yyT3pg+vDgan8DSivq1eS0gwi+KGKE5x8kRcbeFTb/673mkO5SN1IZfmCfA5o+EA==", - "dev": true, "requires": { "@babel/types": "^7.7.4" } @@ -114,7 +99,6 @@ "version": "7.7.4", "resolved": "https://npm.polydev.blue/@babel%2fhelper-split-export-declaration/-/helper-split-export-declaration-7.7.4.tgz", "integrity": "sha512-guAg1SXFcVr04Guk9eq0S4/rWS++sbmyqosJzVs8+1fH5NI+ZcmkaSkc7dmtAFbHFva6yRJnjW3yAcGxjueDug==", - "dev": true, "requires": { "@babel/types": "^7.7.4" } @@ -123,7 +107,6 @@ "version": "7.7.4", "resolved": "https://npm.polydev.blue/@babel%2fhelpers/-/helpers-7.7.4.tgz", "integrity": "sha512-ak5NGZGJ6LV85Q1Zc9gn2n+ayXOizryhjSUBTdu5ih1tlVCJeuQENzc4ItyCVhINVXvIT/ZQ4mheGIsfBkpskg==", - "dev": true, "requires": { "@babel/template": "^7.7.4", "@babel/traverse": "^7.7.4", @@ -134,7 +117,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.0.0.tgz", "integrity": "sha512-UFMC4ZeFC48Tpvj7C8UgLvtkaUuovQX+5xNWrsIoMG8o2z+XFKjKaN9iVmS84dPwVN00W4wPmqvYoZF3EGAsfw==", - "dev": true, "requires": { "chalk": "^2.0.0", "esutils": "^2.0.2", @@ -143,9 +125,8 @@ }, "@babel/parser": { "version": "7.7.5", - "resolved": "https://npm.polydev.blue/@babel%2fparser/-/parser-7.7.5.tgz", - "integrity": "sha512-KNlOe9+/nk4i29g0VXgl8PEXIRms5xKLJeuZ6UptN0fHv+jDiriG+y94X6qAgWTR0h3KaoM1wK5G5h7MHFRSig==", - "dev": true + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.7.5.tgz", + "integrity": "sha512-KNlOe9+/nk4i29g0VXgl8PEXIRms5xKLJeuZ6UptN0fHv+jDiriG+y94X6qAgWTR0h3KaoM1wK5G5h7MHFRSig==" }, "@babel/plugin-syntax-object-rest-spread": { "version": "7.7.4", @@ -160,7 +141,6 @@ "version": "7.7.4", "resolved": "https://npm.polydev.blue/@babel%2ftemplate/-/template-7.7.4.tgz", "integrity": "sha512-qUzihgVPguAzXCK7WXw8pqs6cEwi54s3E+HrejlkuWO6ivMKx9hZl3Y2fSXp9i5HgyWmj7RKP+ulaYnKM4yYxw==", - "dev": true, "requires": { "@babel/code-frame": "^7.0.0", "@babel/parser": "^7.7.4", @@ -171,7 +151,6 @@ "version": "7.7.4", "resolved": "https://npm.polydev.blue/@babel%2ftraverse/-/traverse-7.7.4.tgz", "integrity": "sha512-P1L58hQyupn8+ezVA2z5KBm4/Zr4lCC8dwKCMYzsa5jFMDMQAzaBNy9W5VjB+KAmBjb40U7a/H6ao+Xo+9saIw==", - "dev": true, "requires": { "@babel/code-frame": "^7.5.5", "@babel/generator": "^7.7.4", @@ -188,7 +167,6 @@ "version": "7.5.5", "resolved": "https://npm.polydev.blue/@babel%2fcode-frame/-/code-frame-7.5.5.tgz", "integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==", - "dev": true, "requires": { "@babel/highlight": "^7.0.0" } @@ -196,8 +174,7 @@ "lodash": { "version": "4.17.15", "resolved": "https://npm.polydev.blue/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", - "dev": true + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" } } }, @@ -205,7 +182,6 @@ "version": "7.7.4", "resolved": "https://npm.polydev.blue/@babel%2ftypes/-/types-7.7.4.tgz", "integrity": "sha512-cz5Ji23KCi4T+YIE/BolWosrJuSmoZeN1EFnRtBwF+KKLi8GG/Z2c2hOJJeCXPk4mwk4QFvTmwIodJowXgttRA==", - "dev": true, "requires": { "esutils": "^2.0.2", "lodash": "^4.17.13", @@ -215,8 +191,7 @@ "lodash": { "version": "4.17.15", "resolved": "https://npm.polydev.blue/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", - "dev": true + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" } } }, @@ -1347,7 +1322,6 @@ "version": "1.7.0", "resolved": "https://npm.polydev.blue/convert-source-map/-/convert-source-map-1.7.0.tgz", "integrity": "sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA==", - "dev": true, "requires": { "safe-buffer": "~5.1.1" }, @@ -1355,8 +1329,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://npm.polydev.blue/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } }, @@ -1468,7 +1441,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "dev": true, "requires": { "ms": "^2.1.1" } @@ -2780,8 +2752,7 @@ "globals": { "version": "11.11.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.11.0.tgz", - "integrity": "sha512-WHq43gS+6ufNOEqlrDBxVEbb8ntfXrfAUU2ZOpCxrBdGKW3gyv8mCxAfIBD0DroPKGrJ2eSsXsLtY9MPntsyTw==", - "dev": true + "integrity": "sha512-WHq43gS+6ufNOEqlrDBxVEbb8ntfXrfAUU2ZOpCxrBdGKW3gyv8mCxAfIBD0DroPKGrJ2eSsXsLtY9MPntsyTw==" }, "graceful-fs": { "version": "4.2.3", @@ -3744,8 +3715,7 @@ "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "js-yaml": { "version": "3.13.1", @@ -3814,8 +3784,7 @@ "jsesc": { "version": "2.5.2", "resolved": "https://npm.polydev.blue/jsesc/-/jsesc-2.5.2.tgz", - "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", - "dev": true + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==" }, "json-parse-better-errors": { "version": "1.0.2", @@ -3851,7 +3820,6 @@ "version": "2.1.1", "resolved": "https://npm.polydev.blue/json5/-/json5-2.1.1.tgz", "integrity": "sha512-l+3HXD0GEI3huGq1njuqtzYK8OYJyXMkOLtQ53pjWh89tvWS2h6l+1zMkYWqlb57+SiQodKZyvMEFb2X+KrFhQ==", - "dev": true, "requires": { "minimist": "^1.2.0" } @@ -4188,8 +4156,7 @@ "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", - "dev": true + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, "mute-stream": { "version": "0.0.7", @@ -5204,8 +5171,7 @@ "semver": { "version": "5.7.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", - "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", - "dev": true + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" }, "set-blocking": { "version": "2.0.0", @@ -5722,8 +5688,7 @@ "to-fast-properties": { "version": "2.0.0", "resolved": "https://npm.polydev.blue/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", - "dev": true + "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=" }, "to-object-path": { "version": "0.3.0", diff --git a/package.json b/package.json index 9b1e86e..f0d59a8 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "gettext-extract": "./src/extract-cli.js" }, "dependencies": { + "@babel/core": "^7.7.5", "@vue/component-compiler-utils": "^2.6.0", "acorn": "^6.4.0", "acorn-stage3": "^2.0.0", diff --git a/src/extract-cli.js b/src/extract-cli.js index b6cf22f..936e990 100755 --- a/src/extract-cli.js +++ b/src/extract-cli.js @@ -23,9 +23,10 @@ const extraAttribute = argv.attribute || false; const extraFilter = argv.filter || false; const removeHTMLWhitespaces = argv.removeHTMLWhitespaces || false; const filterPrefix = argv.filterPrefix || constants.DEFAULT_FILTER_PREFIX; +const jsParser = argv.parser || 'auto'; if (!quietMode && (!files || files.length === 0)) { - console.log('Usage:\n\tgettext-extract [--attribute EXTRA-ATTRIBUTE] [--filterPrefix FILTER-PREFIX] [--output OUTFILE] '); + console.log('Usage:\n\tgettext-extract [--attribute EXTRA-ATTRIBUTE] [--filterPrefix FILTER-PREFIX] [--output OUTFILE] [--parser auto|acorn|babel] '); process.exit(1); } @@ -63,7 +64,7 @@ files.forEach(function(filename) { console.log(`[${PROGRAM_NAME}] will not extract: '${filename}' (invalid extension)`); return; } - console.log(`[${PROGRAM_NAME}] extracting: '${filename}`); + console.log(`[${PROGRAM_NAME}] extracting: '${filename}'`); try { let data = fs.readFileSync(file, {encoding: 'utf-8'}).toString(); extractor.parse(file, extract.preprocessTemplate(data, ext)); @@ -80,10 +81,10 @@ files.forEach(function(filename) { } if (lang === 'js') { - extractor.parseJavascript(file, data); + extractor.parseJavascript(file, data, jsParser); } } catch (e) { - console.error(`[${PROGRAM_NAME}] could not read: '${filename}`); + console.error(`[${PROGRAM_NAME}] could not read: '${filename}' using parser ${jsParser}`); console.trace(e); process.exit(1); } diff --git a/src/extract.js b/src/extract.js index aa4dc5a..2d47581 100644 --- a/src/extract.js +++ b/src/extract.js @@ -261,10 +261,10 @@ exports.Extractor = class Extractor { } } - parseJavascript(filename, content) { + parseJavascript(filename, content, parser='auto') { const jsContent = flowRemoveTypes(content).toString(); - const extractedStringsFromScript = jsExtractor.extractStringsFromJavascript(filename, jsContent); + const extractedStringsFromScript = jsExtractor.extractStringsFromJavascript(filename, jsContent, parser); this.processStrings(extractedStringsFromScript); } diff --git a/src/javascript-extract.js b/src/javascript-extract.js index f4cbf09..fc9edd3 100644 --- a/src/javascript-extract.js +++ b/src/javascript-extract.js @@ -1,6 +1,7 @@ const {Parser} = require('acorn'); const stage3 = require('acorn-stage3'); const Pofile = require('pofile'); +const babel = require('@babel/core'); const {MARKER_NO_CONTEXT, DEFAULT_VUE_GETTEXT_FUNCTIONS} = require('./constants.js'); @@ -33,20 +34,42 @@ function extractConcatenatedStrings(value, allTokens, index) { } -function getGettextEntriesFromScript(script) { +function getGettextEntriesFromScript(script, parser = 'auto') { + const allTokens = []; - const ACORN_OPTIONS = { - ecmaVersion: 10, - sourceType: 'module', - locations: true, - onToken: allTokens, - plugins: { - stage3: true, - }, - }; + switch (parser) { + case 'auto': + try { + return getGettextEntriesFromScript(script, 'acorn'); + } catch (e) { + return getGettextEntriesFromScript(script, 'babel'); + } + break; + case 'acorn': + const ACORN_OPTIONS = { + ecmaVersion: 10, + sourceType: 'module', + locations: true, + onToken: allTokens, + plugins: { + stage3: true, + }, + }; + + Parser.extend(stage3).parse(script, ACORN_OPTIONS); + break; + case 'babel': + const babelResult = babel.parseSync(script, { + sourceType: 'module', + parserOpts: { + tokens: true, + } + }); + allTokens.push(...babelResult.tokens); + break; + } - Parser.extend(stage3).parse(script, ACORN_OPTIONS); let extractedEntries = []; @@ -104,8 +127,8 @@ function getGettextEntriesFromScript(script) { return extractedEntries; } -function extractStringsFromJavascript(filename, script) { - const gettextEntries = getGettextEntriesFromScript(script); +function extractStringsFromJavascript(filename, script, parser = 'auto') { + const gettextEntries = getGettextEntriesFromScript(script, parser); return gettextEntries.map((entry) => { return Object.assign(