cli: add --plugins flag to load from the command line (#7407)

Author: brendankenny

lighthouse-cli/cli-flags.js

@@ -104,6 +104,7 @@ function getFlags(manualArgv) {
         'only-audits': 'Only run the specified audits',
         'only-categories': 'Only run the specified categories',
         'skip-audits': 'Run everything except these audits',
+        'plugins': 'Run the specified plugins',
         'print-config': 'Print the normalized config for the given config and options, then exit.',
       })
       // set aliases
@@ -136,6 +137,7 @@ function getFlags(manualArgv) {
       .array('onlyCategories')
       .array('skipAudits')
       .array('output')
+      .array('plugins')
       .string('extraHeaders')
       .string('channel')
       .string('precomputedLanternDataPath')

lighthouse-cli/test/cli/run-test.js

@@ -19,42 +19,73 @@ const fastConfig = {
   },
 };
 
+// Map plugin name to fixture since not actually installed in node_modules/.
+jest.mock('lighthouse-plugin-simple', () => {
+  return require('../../../lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-simple/');
+}, {virtual: true});
+
 const getFlags = require('../../cli-flags').getFlags;
 
 describe('CLI run', function() {
-  it('runLighthouse completes a LH round trip', () => {
-    const url = 'chrome://version';
+  describe('LH round trip', () => {
+    /** @type {LH.RunnerResult} */
+    let passedResults;
     const filename = path.join(process.cwd(), 'run.ts.results.json');
-    const timeoutFlag = `--max-wait-for-load=${9000}`;
-    const flags = getFlags(`--output=json --output-path=${filename} ${timeoutFlag} ${url}`);
-    return run.runLighthouse(url, flags, fastConfig).then(passedResults => {
-      if (!passedResults) {
-        assert.fail('no results');
-        return;
+    /** @type {LH.Result} */
+    let fileResults;
+
+    beforeAll(async () => {
+      const url = 'chrome://version';
+      const timeoutFlag = `--max-wait-for-load=${9000}`;
+      const pluginsFlag = '--plugins=lighthouse-plugin-simple';
+
+      // eslint-disable-next-line max-len
+      const flags = getFlags(`--output=json --output-path=${filename} ${pluginsFlag} ${timeoutFlag} ${url}`);
+
+      const rawResult = await run.runLighthouse(url, flags, fastConfig);
+
+      if (!rawResult) {
+        return assert.fail('no results');
       }
+      passedResults = rawResult;
 
-      const {lhr} = passedResults;
       assert.ok(fs.existsSync(filename));
-      /** @type {LH.Result} */
-      const results = JSON.parse(fs.readFileSync(filename, 'utf-8'));
-      assert.equal(results.audits.viewport.rawValue, false);
+      fileResults = JSON.parse(fs.readFileSync(filename, 'utf-8'));
+    }, 20 * 1000);
+
+    afterAll(() => {
+      fs.unlinkSync(filename);
+    });
+
+    it('returns results that match the saved results', () => {
+      const {lhr} = passedResults;
+      assert.equal(fileResults.audits.viewport.rawValue, false);
 
       // passed results match saved results
-      assert.strictEqual(results.fetchTime, lhr.fetchTime);
-      assert.strictEqual(results.requestedUrl, lhr.requestedUrl);
-      assert.strictEqual(results.finalUrl, lhr.finalUrl);
-      assert.strictEqual(results.audits.viewport.rawValue, lhr.audits.viewport.rawValue);
+      assert.strictEqual(fileResults.fetchTime, lhr.fetchTime);
+      assert.strictEqual(fileResults.requestedUrl, lhr.requestedUrl);
+      assert.strictEqual(fileResults.finalUrl, lhr.finalUrl);
+      assert.strictEqual(fileResults.audits.viewport.rawValue, lhr.audits.viewport.rawValue);
       assert.strictEqual(
-          Object.keys(results.audits).length,
+          Object.keys(fileResults.audits).length,
           Object.keys(lhr.audits).length);
-      assert.deepStrictEqual(results.timing, lhr.timing);
-      assert.ok(results.timing.total !== 0);
+      assert.deepStrictEqual(fileResults.timing, lhr.timing);
+    });
+
+    it('includes timing information', () => {
+      assert.ok(passedResults.lhr.timing.total !== 0);
+    });
 
-      assert.equal(results.configSettings.channel, 'cli');
+    it('correctly sets the channel', () => {
+      assert.equal(passedResults.lhr.configSettings.channel, 'cli');
+    });
 
-      fs.unlinkSync(filename);
+    it('merged the plugin into the config', () => {
+      // Audits have been pruned because of onlyAudits, but groups get merged in.
+      const groupNames = Object.keys(passedResults.lhr.categoryGroups || {});
+      assert.ok(groupNames.includes('lighthouse-plugin-simple-new-group'));
     });
-  }, 20 * 1000);
+  });
 });
 
 describe('flag coercing', () => {

lighthouse-core/config/config.js

@@ -362,7 +362,7 @@ class Config {
     const configDir = configPath ? path.dirname(configPath) : undefined;
 
     // Validate and merge in plugins (if any).
-    configJSON = Config.mergePlugins(configJSON, configDir);
+    configJSON = Config.mergePlugins(configJSON, flags, configDir);
 
     const settings = Config.initSettings(configJSON.settings, flags);
 
@@ -459,22 +459,23 @@ class Config {
 
   /**
    * @param {LH.Config.Json} configJSON
+   * @param {LH.Flags=} flags
    * @param {string=} configDir
    * @return {LH.Config.Json}
    */
-  static mergePlugins(configJSON, configDir) {
-    const pluginNames = configJSON.plugins;
+  static mergePlugins(configJSON, flags, configDir) {
+    const configPlugins = configJSON.plugins || [];
+    const flagPlugins = (flags && flags.plugins) || [];
+    const pluginNames = new Set([...configPlugins, ...flagPlugins]);
 
-    if (pluginNames) {
-      for (const pluginName of pluginNames) {
-        assertValidPluginName(configJSON, pluginName);
+    for (const pluginName of pluginNames) {
+      assertValidPluginName(configJSON, pluginName);
 
-        const pluginPath = Config.resolveModule(pluginName, configDir, 'plugin');
-        const rawPluginJson = require(pluginPath);
-        const pluginJson = ConfigPlugin.parsePlugin(rawPluginJson, pluginName);
+      const pluginPath = Config.resolveModule(pluginName, configDir, 'plugin');
+      const rawPluginJson = require(pluginPath);
+      const pluginJson = ConfigPlugin.parsePlugin(rawPluginJson, pluginName);
 
-        configJSON = Config.extendConfigJSON(configJSON, pluginJson);
-      }
+      configJSON = Config.extendConfigJSON(configJSON, pluginJson);
     }
 
     return configJSON;

lighthouse-core/test/config/config-test.js

@@ -697,19 +697,33 @@ describe('Config', () => {
   });
 
   describe('mergePlugins', () => {
+    // Include a configPath flag so that config.js looks for the plugins in the fixtures dir.
     const configFixturePath = __dirname + '/../fixtures/config-plugins/';
 
-    it('should append audits and a group', () => {
+    it('should append audits', () => {
       const configJson = {
         audits: ['installable-manifest', 'metrics'],
         plugins: ['lighthouse-plugin-simple'],
       };
       const config = new Config(configJson, {configPath: configFixturePath});
-      const groupIds = Object.keys(config.groups);
       assert.deepStrictEqual(config.audits.map(a => a.path),
         ['installable-manifest', 'metrics', 'redirects', 'user-timings']);
-      assert.ok(groupIds.length === 1);
-      assert.strictEqual(groupIds[groupIds.length - 1], 'lighthouse-plugin-simple-new-group');
+    });
+
+    it('should append and use plugin-prefixed groups', () => {
+      const configJson = {
+        audits: ['installable-manifest', 'metrics'],
+        plugins: ['lighthouse-plugin-simple'],
+        groups: {
+          configGroup: {title: 'This is a group in the base config'},
+        },
+      };
+      const config = new Config(configJson, {configPath: configFixturePath});
+
+      const groupIds = Object.keys(config.groups);
+      assert.ok(groupIds.length === 2);
+      assert.strictEqual(groupIds[0], 'configGroup');
+      assert.strictEqual(groupIds[1], 'lighthouse-plugin-simple-new-group');
       assert.strictEqual(config.groups['lighthouse-plugin-simple-new-group'].title, 'New Group');
       assert.strictEqual(config.categories['lighthouse-plugin-simple'].auditRefs[0].group,
         'lighthouse-plugin-simple-new-group');
@@ -727,6 +741,39 @@ describe('Config', () => {
       assert.strictEqual(config.categories['lighthouse-plugin-simple'].title, 'Simple');
     });
 
+    it('should load plugins from the config and from passed-in flags', () => {
+      const baseConfigJson = {
+        audits: ['installable-manifest'],
+        categories: {
+          myManifest: {
+            auditRefs: [{id: 'installable-manifest', weight: 9000}],
+          },
+        },
+      };
+      const baseFlags = {configPath: configFixturePath};
+      const simplePluginName = 'lighthouse-plugin-simple';
+      const noGroupsPluginName = 'lighthouse-plugin-no-groups';
+
+      const allConfigConfigJson = {...baseConfigJson, plugins: [simplePluginName,
+        noGroupsPluginName]};
+      const allPluginsInConfigConfig = new Config(allConfigConfigJson, baseFlags);
+
+      const allFlagsFlags = {...baseFlags, plugins: [simplePluginName, noGroupsPluginName]};
+      const allPluginsInFlagsConfig = new Config(baseConfigJson, allFlagsFlags);
+
+      const mixedConfigJson = {...baseConfigJson, plugins: [simplePluginName]};
+      const mixedFlags = {...baseFlags, plugins: [noGroupsPluginName]};
+      const pluginsInConfigAndFlagsConfig = new Config(mixedConfigJson, mixedFlags);
+
+      // Double check that we're not comparing empty objects.
+      const categoryNames = Object.keys(allPluginsInConfigConfig.categories);
+      assert.deepStrictEqual(categoryNames,
+        ['myManifest', 'lighthouse-plugin-simple', 'lighthouse-plugin-no-groups']);
+
+      assert.deepStrictEqual(allPluginsInConfigConfig, allPluginsInFlagsConfig);
+      assert.deepStrictEqual(allPluginsInConfigConfig, pluginsInConfigAndFlagsConfig);
+    });
+
     it('should throw if the plugin is invalid', () => {
       const configJson = {
         extends: 'lighthouse:default',

lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-no-groups/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "lighthouse-plugin-no-groups",
+  "private": true,
+  "main": "./plugin-no-groups.js"
+}

lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-no-groups/plugin-no-groups.js

@@ -0,0 +1,19 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/** @type {LH.Config.Plugin} */
+module.exports = {
+  audits: [
+    {path: 'uses-rel-preload'},
+  ],
+  category: {
+    title: 'NoGroups',
+    auditRefs: [
+      {id: 'uses-rel-preload', weight: 1},
+    ],
+  },
+};

types/externs.d.ts

@@ -142,6 +142,8 @@ declare global {
       logLevel?: 'silent'|'error'|'info'|'verbose';
       /** The path to the config JSON. */
       configPath?: string;
+      /** Run the specified plugins. */
+      plugins?: string[];
     }
 
     /**