feat(validation): warn about unknown stryker config options

packages/api/schema/stryker-core.json

@@ -180,6 +180,18 @@
       "required": [
         "name"
       ]
+    },
+    "warningOptions": {
+      "title": "WarningOptions",
+      "type": "object",
+      "default": {},
+      "properties": {
+        "unknownOptions": {
+          "description": "decide whether or not to log warnings when additional stryker options are configured",
+          "type": "boolean",
+          "default": true
+        }
+      }
     }
   },
   "properties": {
@@ -328,6 +340,18 @@
         "type": "string"
       },
       "default": []
+    },
+    "warnings": {
+      "default": true,
+      "oneOf": [
+        {
+          "type": "boolean"
+        },
+        {
+          "$ref": "#/definitions/warningOptions"
+        }
+      ],
+      "description": "Enable or disable certain warnings"
     }
   }
 }

packages/api/testResources/module/useCore.ts

@@ -38,6 +38,7 @@ const optionsAllArgs: StrykerOptions = {
     baseDir: 'mydir'
   },
   tempDirName: '.stryker-tmp',
+  warnings: true
 };
 
 const textFile: File = new File('foo/bar.js', Buffer.from('foobar'));

packages/core/src/config/OptionsValidator.ts

@@ -1,11 +1,12 @@
 import Ajv = require('ajv');
-import { StrykerOptions, strykerCoreSchema } from '@stryker-mutator/api/core';
+import { StrykerOptions, strykerCoreSchema, WarningOptions } from '@stryker-mutator/api/core';
 import { tokens, commonTokens } from '@stryker-mutator/api/plugin';
-import { noopLogger, normalizeWhitespaces } from '@stryker-mutator/util';
+import { noopLogger, normalizeWhitespaces, propertyPath } from '@stryker-mutator/util';
 import { Logger } from '@stryker-mutator/api/logging';
 
 import { coreTokens } from '../di';
 import { ConfigError } from '../errors';
+import { isWarningEnabled } from '../utils/objectUtils';
 
 import { describeErrors } from './validationErrors';
 
@@ -68,8 +69,33 @@ export function defaultOptions(): StrykerOptions {
   return options;
 }
 
+validateOptions.inject = tokens(commonTokens.options, coreTokens.optionsValidator);
 export function validateOptions(options: unknown, optionsValidator: OptionsValidator): StrykerOptions {
   optionsValidator.validate(options);
   return options;
 }
-validateOptions.inject = tokens(commonTokens.options, coreTokens.optionsValidator);
+
+markUnknownOptions.inject = tokens(commonTokens.options, coreTokens.validationSchema, commonTokens.logger);
+export function markUnknownOptions(options: StrykerOptions, schema: object, log: Logger): StrykerOptions {
+  const OPTIONS_ADDED_BY_STRYKER = ['set', 'configFile', '$schema'];
+  if (isWarningEnabled('unknownOptions', options.warnings)) {
+    const unknownPropertyNames = Object.keys(options)
+      .filter((key) => !key.endsWith('_comment'))
+      .filter((key) => !OPTIONS_ADDED_BY_STRYKER.includes(key))
+      .filter((key) => !Object.keys((schema as any).properties).includes(key));
+    unknownPropertyNames.forEach((unknownPropertyName) => {
+      log.warn(`Unknown stryker config option "${unknownPropertyName}".`);
+    });
+    const p = `${propertyPath<StrykerOptions>('warnings')}.${propertyPath<WarningOptions>('unknownOptions')}`;
+    if (unknownPropertyNames.length) {
+      log.warn(`
+   Possible causes:
+   * Is it a typo on your end?
+   * Did you only write this property as a comment? If so, please postfix it with "_comment".
+   * You might be missing a plugin that is supposed to use it. Stryker loaded plugins from: ${JSON.stringify(options.plugins)}
+   * The plugin that is using it did not contribute explicit validation. 
+   (disable "${p}" to ignore this warning)`);
+    }
+  }
+  return options;
+}

packages/core/src/di/buildMainInjector.ts

@@ -5,7 +5,14 @@ import { TestFramework } from '@stryker-mutator/api/test_framework';
 import { getLogger } from 'log4js';
 import { rootInjector } from 'typed-inject';
 
-import { OptionsEditorApplier, readConfig, buildSchemaWithPluginContributions, OptionsValidator, validateOptions } from '../config';
+import {
+  OptionsEditorApplier,
+  readConfig,
+  buildSchemaWithPluginContributions,
+  OptionsValidator,
+  validateOptions,
+  markUnknownOptions,
+} from '../config';
 import ConfigReader from '../config/ConfigReader';
 import BroadcastReporter from '../reporters/BroadcastReporter';
 import { TemporaryDirectory } from '../utils/TemporaryDirectory';
@@ -26,10 +33,29 @@ export interface MainContext extends OptionsContext {
   [coreTokens.temporaryDirectory]: TemporaryDirectory;
 }
 
+type BasicInjector = Injector<Pick<MainContext, 'logger' | 'getLogger'>>;
+type PluginResolverInjector = Injector<Pick<MainContext, 'logger' | 'getLogger' | 'options' | 'pluginResolver'>>;
+
 export function buildMainInjector(cliOptions: Partial<StrykerOptions>): Injector<MainContext> {
-  return rootInjector
-    .provideValue(commonTokens.getLogger, getLogger)
-    .provideFactory(commonTokens.logger, loggerFactory, Scope.Transient)
+  const basicInjector = createBasicInjector();
+  const pluginResolverInjector = createPluginResolverInjector(cliOptions, basicInjector);
+  return pluginResolverInjector
+    .provideFactory(commonTokens.mutatorDescriptor, mutatorDescriptorFactory)
+    .provideFactory(coreTokens.pluginCreatorReporter, PluginCreator.createFactory(PluginKind.Reporter))
+    .provideFactory(coreTokens.pluginCreatorTestFramework, PluginCreator.createFactory(PluginKind.TestFramework))
+    .provideFactory(coreTokens.pluginCreatorMutator, PluginCreator.createFactory(PluginKind.Mutator))
+    .provideClass(coreTokens.reporter, BroadcastReporter)
+    .provideFactory(coreTokens.testFramework, testFrameworkFactory)
+    .provideClass(coreTokens.temporaryDirectory, TemporaryDirectory)
+    .provideClass(coreTokens.timer, Timer);
+}
+
+function createBasicInjector(): BasicInjector {
+  return rootInjector.provideValue(commonTokens.getLogger, getLogger).provideFactory(commonTokens.logger, loggerFactory, Scope.Transient);
+}
+
+export function createPluginResolverInjector(cliOptions: Partial<StrykerOptions>, parent: BasicInjector): PluginResolverInjector {
+  return parent
     .provideValue(coreTokens.cliOptions, cliOptions)
     .provideValue(coreTokens.validationSchema, strykerCoreSchema)
     .provideClass(coreTokens.optionsValidator, OptionsValidator)
@@ -40,18 +66,11 @@ export function buildMainInjector(cliOptions: Partial<StrykerOptions>): Injector
     .provideFactory(coreTokens.validationSchema, buildSchemaWithPluginContributions)
     .provideClass(coreTokens.optionsValidator, OptionsValidator)
     .provideFactory(commonTokens.options, validateOptions)
+    .provideFactory(commonTokens.options, markUnknownOptions)
     .provideFactory(coreTokens.pluginCreatorConfigEditor, PluginCreator.createFactory(PluginKind.ConfigEditor))
     .provideFactory(coreTokens.pluginCreatorOptionsEditor, PluginCreator.createFactory(PluginKind.OptionsEditor))
     .provideClass(coreTokens.configOptionsApplier, OptionsEditorApplier)
-    .provideFactory(commonTokens.options, applyOptionsEditors)
-    .provideFactory(commonTokens.mutatorDescriptor, mutatorDescriptorFactory)
-    .provideFactory(coreTokens.pluginCreatorReporter, PluginCreator.createFactory(PluginKind.Reporter))
-    .provideFactory(coreTokens.pluginCreatorTestFramework, PluginCreator.createFactory(PluginKind.TestFramework))
-    .provideFactory(coreTokens.pluginCreatorMutator, PluginCreator.createFactory(PluginKind.Mutator))
-    .provideClass(coreTokens.reporter, BroadcastReporter)
-    .provideFactory(coreTokens.testFramework, testFrameworkFactory)
-    .provideClass(coreTokens.temporaryDirectory, TemporaryDirectory)
-    .provideClass(coreTokens.timer, Timer);
+    .provideFactory(commonTokens.options, applyOptionsEditors);
 }
 
 function pluginDescriptorsFactory(options: StrykerOptions): readonly string[] {

packages/core/src/utils/objectUtils.ts

@@ -1,5 +1,6 @@
 import treeKill = require('tree-kill');
-import { StrykerError } from '@stryker-mutator/util';
+import { StrykerError, KnownKeys } from '@stryker-mutator/util';
+import { WarningOptions } from '@stryker-mutator/api/core';
 
 export { serialize, deserialize } from 'surrial';
 
@@ -26,6 +27,14 @@ export function getEnvironmentVariableOrThrow(name: string): string {
   }
 }
 
+export function isWarningEnabled(warningType: KnownKeys<WarningOptions>, warningOptions: WarningOptions | boolean): boolean {
+  if (typeof warningOptions === 'boolean') {
+    return warningOptions;
+  } else {
+    return !!warningOptions[warningType];
+  }
+}
+
 /**
  * A wrapper around `process.exitCode = n` (for testability)
  */

packages/core/stryker.conf.js

@@ -2,4 +2,4 @@ const path = require('path');
 const settings = require('../../stryker.parent.conf');
 const moduleName = __dirname.split(path.sep).pop();
 settings.dashboard.module = moduleName;
-module.exports = settings;
+module.exports = settings;

packages/core/test/helpers/testUtils.ts

@@ -1,5 +1,11 @@
+import { resolve } from 'path';
+
 export function sleep(ms = 0) {
   return new Promise((res) => {
     setTimeout(res, ms);
   });
 }
+
+export function resolveFromRoot(...pathSegments: string[]) {
+  return resolve(__dirname, '..', '..', ...pathSegments);
+}

packages/core/test/integration/options-validation/options-validation.it.spec.ts

@@ -0,0 +1,21 @@
+import { expect } from 'chai';
+import { testInjector } from '@stryker-mutator/test-helpers';
+import { commonTokens } from '@stryker-mutator/api/plugin';
+
+import { createPluginResolverInjector } from '../../../src/di';
+import { resolveFromRoot } from '../../helpers/testUtils';
+
+import sinon = require('sinon');
+
+describe('Options validation integration', () => {
+  it('should log about unknown properties in log file', () => {
+    const optionsProvider = createPluginResolverInjector(
+      {
+        configFile: resolveFromRoot('testResources', 'options-validation', 'unknown-options.conf.json'),
+      },
+      testInjector.injector
+    );
+    optionsProvider.resolve(commonTokens.options);
+    expect(testInjector.logger.warn).calledWithMatch(sinon.match('Unknown stryker config option "this is an unknown property"'));
+  });
+});

packages/core/test/unit/config/OptionsValidator.spec.ts

@@ -1,9 +1,10 @@
+import sinon = require('sinon');
 import { strykerCoreSchema, StrykerOptions } from '@stryker-mutator/api/core';
-import { testInjector } from '@stryker-mutator/test-helpers';
+import { testInjector, factory } from '@stryker-mutator/test-helpers';
 import { expect } from 'chai';
 import { normalizeWhitespaces } from '@stryker-mutator/util';
 
-import { OptionsValidator } from '../../../src/config/OptionsValidator';
+import { OptionsValidator, validateOptions, markUnknownOptions } from '../../../src/config/OptionsValidator';
 import { coreTokens } from '../../../src/di';
 
 describe(OptionsValidator.name, () => {
@@ -221,3 +222,62 @@ describe(OptionsValidator.name, () => {
     }
   }
 });
+
+describe(validateOptions.name, () => {
+  let optionsValidatorMock: sinon.SinonStubbedInstance<OptionsValidator>;
+
+  beforeEach(() => {
+    optionsValidatorMock = sinon.createStubInstance(OptionsValidator);
+  });
+
+  it('should validate the options using given optionsValidator', () => {
+    const options = { foo: 'bar' };
+    const output = validateOptions(options, (optionsValidatorMock as unknown) as OptionsValidator);
+    expect(options).eq(output);
+    expect(optionsValidatorMock.validate).calledWith(options);
+  });
+});
+
+describe(markUnknownOptions.name, () => {
+  it('should not warn when there are no unknown properties', () => {
+    testInjector.options.htmlReporter = {
+      baseDir: 'test',
+    };
+    expect(testInjector.logger.warn).not.called;
+  });
+
+  it('should return the options, no matter what', () => {
+    testInjector.options['this key does not exist'] = 'foo';
+    const output = markUnknownOptions(testInjector.options, strykerCoreSchema, testInjector.logger);
+    expect(output).eq(testInjector.options);
+  });
+
+  it('should not warn when unknown properties are postfixed with "_comment"', () => {
+    testInjector.options['maxConcurrentTestRunners_comment'] = 'Recommended to use half of your cores';
+    markUnknownOptions(testInjector.options, strykerCoreSchema, testInjector.logger);
+    expect(testInjector.logger.warn).not.called;
+  });
+
+  it('should warn about unknown properties', () => {
+    testInjector.options['karma'] = {};
+    testInjector.options['jest'] = {};
+    markUnknownOptions(testInjector.options, strykerCoreSchema, testInjector.logger);
+    expect(testInjector.logger.warn).calledThrice;
+    expect(testInjector.logger.warn).calledWith('Unknown stryker config option "karma".');
+    expect(testInjector.logger.warn).calledWith('Unknown stryker config option "jest".');
+    expect(testInjector.logger.warn).calledWithMatch('Possible causes');
+  });
+  it('should not warn about unknown properties when warnings are disabled', () => {
+    testInjector.options['karma'] = {};
+    testInjector.options.warnings = factory.warningOptions({ unknownOptions: false });
+    markUnknownOptions(testInjector.options, strykerCoreSchema, testInjector.logger);
+    expect(testInjector.logger.warn).not.called;
+  });
+  it('should ignore options added by Stryker itself', () => {
+    testInjector.options['set'] = {};
+    testInjector.options['configFile'] = {};
+    testInjector.options['$schema'] = '';
+    markUnknownOptions(testInjector.options, strykerCoreSchema, testInjector.logger);
+    expect(testInjector.logger.warn).not.called;
+  });
+});

packages/core/test/unit/di/buildMainInjector.spec.ts

@@ -21,11 +21,11 @@ describe(buildMainInjector.name, () => {
   let testFrameworkMock: TestFramework;
   let configReaderMock: sinon.SinonStubbedInstance<ConfigReader>;
   let pluginCreatorMock: sinon.SinonStubbedInstance<PluginCreator<any>>;
-  let buildSchemaWithPluginContributionsStub: sinon.SinonStub;
   let optionsEditorApplierMock: sinon.SinonStubbedInstance<configModule.OptionsEditorApplier>;
   let broadcastReporterMock: sinon.SinonStubbedInstance<Reporter>;
   let optionsValidatorStub: sinon.SinonStubbedInstance<configModule.OptionsValidator>;
   let expectedConfig: StrykerOptions;
+  let validationSchemaContributions: object[];
 
   beforeEach(() => {
     configReaderMock = sinon.createStubInstance(ConfigReader);
@@ -36,14 +36,14 @@ describe(buildMainInjector.name, () => {
     testFrameworkOrchestratorMock = sinon.createStubInstance(TestFrameworkOrchestrator);
     testFrameworkOrchestratorMock.determineTestFramework.returns(testFrameworkMock);
     pluginLoaderMock = sinon.createStubInstance(di.PluginLoader);
+    validationSchemaContributions = [];
+    pluginLoaderMock.resolveValidationSchemaContributions.returns(validationSchemaContributions);
     optionsValidatorStub = sinon.createStubInstance(configModule.OptionsValidator);
-    buildSchemaWithPluginContributionsStub = sinon.stub();
     expectedConfig = factory.strykerOptions();
     broadcastReporterMock = factory.reporter('broadcast');
     configReaderMock.readConfig.returns(expectedConfig);
     stubInjectable(PluginCreator, 'createFactory').returns(() => pluginCreatorMock);
     stubInjectable(configModule, 'OptionsEditorApplier').returns(optionsEditorApplierMock);
-    stubInjectable(configModule, 'buildSchemaWithPluginContributions').returns(buildSchemaWithPluginContributionsStub);
     stubInjectable(configModule, 'OptionsValidator').returns(optionsValidatorStub);
     stubInjectable(di, 'PluginLoader').returns(pluginLoaderMock);
     stubInjectable(configReaderModule, 'default').returns(configReaderMock);
@@ -97,6 +97,12 @@ describe(buildMainInjector.name, () => {
       buildMainInjector({}).resolve(commonTokens.options);
       expect(optionsValidatorStub.validate).calledWith(expectedConfig);
     });
+
+    it('should warn about unknown properties', () => {
+      expectedConfig.foo = 'bar';
+      buildMainInjector({}).resolve(commonTokens.options);
+      expect(currentLogMock().warn).calledWithMatch('Unknown stryker config option "foo"');
+    });
   });
 
   it('should supply mutatorDescriptor', () => {

packages/core/testResources/options-validation/unknown-options.conf.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://raw.githubusercontent.com/stryker-mutator/stryker/master/packages/api/schema/stryker-core.json",
+  "plugins": [],
+  "this is an unknown property": "foo"
+}

packages/test-helpers/src/factory.ts

@@ -1,6 +1,14 @@
 import Ajv = require('ajv');
 import { ConfigEditor } from '@stryker-mutator/api/config';
-import { File, Location, MutationScoreThresholds, StrykerOptions, MutatorDescriptor, strykerCoreSchema } from '@stryker-mutator/api/core';
+import {
+  File,
+  Location,
+  MutationScoreThresholds,
+  StrykerOptions,
+  MutatorDescriptor,
+  strykerCoreSchema,
+  WarningOptions,
+} from '@stryker-mutator/api/core';
 import { Logger } from '@stryker-mutator/api/logging';
 import { Mutant } from '@stryker-mutator/api/mutant';
 import { MatchedMutant, MutantResult, MutantStatus, mutationTestReportSchema, Reporter } from '@stryker-mutator/api/report';
@@ -49,6 +57,10 @@ export function pluginResolver(): sinon.SinonStubbedInstance<PluginResolver> {
   };
 }
 
+export const warningOptions = factoryMethod<WarningOptions>(() => ({
+  unknownOptions: true,
+}));
+
 export const mutantResult = factoryMethod<MutantResult>(() => ({
   id: '256',
   location: location(),

packages/util/src/KnownKeys.ts

@@ -0,0 +1,9 @@
+/**
+ * Known keys filters out the index signature from the keys of a type
+ * @see https://stackoverflow.com/questions/51465182/typescript-remove-index-signature-using-mapped-types
+ */
+export type KnownKeys<T> = {
+  [K in keyof T]: string extends K ? never : number extends K ? never : K;
+} extends { [_ in keyof T]: infer U }
+  ? U
+  : never;

packages/util/src/index.ts

@@ -2,6 +2,7 @@ export { default as childProcessAsPromised } from './childProcessAsPromised';
 export { default as StrykerError } from './StrykerError';
 export * from './errors';
 export * from './Immutable';
+export * from './KnownKeys';
 export * from './stringUtils';
 export * from './noopLogger';
 export * from './notEmpty';