From 464e59372a52eacbe753c8e51b7cbc0db4fc0f4b Mon Sep 17 00:00:00 2001 From: Pine Wu Date: Thu, 23 May 2019 17:02:41 -0700 Subject: [PATCH 1/5] vscode-test related updates --- .../continuous-integration.md | 132 ++++------- .../images/testing-extension/debug.mp4 | 3 + .../images/testing-extension/launch-tests.png | 3 - .../testing-extension/pipelines-logo.png | 3 - .../images/testing-extension/pipelines.png | 3 - .../images/testing-extension/test-output.png | 3 - .../testing-extension.md | 210 +++++++++++++----- release-notes/v1_35.md | 23 ++ 8 files changed, 227 insertions(+), 153 deletions(-) create mode 100644 api/working-with-extensions/images/testing-extension/debug.mp4 delete mode 100644 api/working-with-extensions/images/testing-extension/launch-tests.png delete mode 100644 api/working-with-extensions/images/testing-extension/pipelines-logo.png delete mode 100644 api/working-with-extensions/images/testing-extension/pipelines.png delete mode 100644 api/working-with-extensions/images/testing-extension/test-output.png diff --git a/api/working-with-extensions/continuous-integration.md b/api/working-with-extensions/continuous-integration.md index 83b4176444..6a04900fd5 100644 --- a/api/working-with-extensions/continuous-integration.md +++ b/api/working-with-extensions/continuous-integration.md @@ -9,24 +9,7 @@ MetaDescription: Use Continuous Integration for testing Visual Studio Code exten # Continuous Integration -Extension tests can be run on CI services. The `vscode` npm module provides a built-in command (`bin/test`) which: - -1. Downloads and unzips VS Code; -2. Launches your extension tests inside VS Code; -3. Prints the results to the console and exits with an appropriate status code. - -The command will expose some optional environment variables, which you can use to customize the build: - -| Name | Description | -| ------------------------- | ---------------------------------------------------------------------------------------------- | -| `CODE_VERSION` | Version of VS Code to run the tests against (e.g. `0.10.10`) | -| `CODE_DOWNLOAD_URL` | Full URL of a VS Code drop to use for running tests against | -| `CODE_TESTS_PATH` | Location of the tests to execute (default is `process.cwd()/out/test` or `process.cwd()/test`) | -| `CODE_EXTENSIONS_PATH` | Location of the extensions to load (default is `process.cwd()`) | -| `CODE_TESTS_WORKSPACE` | Location of a workspace to open for the test instance (default is CODE_TESTS_PATH) | -| `CODE_LOCALE` | Display language to use when running the tests (default is English) | -| `CODE_DISABLE_EXTENSIONS` | Disable all other extensions except the one that is being tested | -| `CODE_TESTS_DATA_DIR` | Allows to specify the user-data-dir for the tests to use and thus enables to run multiple tests at the same time | +Extension tests can be run on CI services. The `vscode-test` repository itself contains a sample extension that is tested on Azure Devops Pipelines. You can check out the [build pipeline](https://dev.azure.com/vscode/VSCode/_build?definitionId=14) or jump directly to the [build definition yaml file](https://github.com/microsoft/vscode-test/blob/master/azure-pipelines.yml). ## Azure Pipelines @@ -34,83 +17,50 @@ The command will expose some optional environment variables, which you can use t You can create free projects on [Azure DevOps](https://azure.microsoft.com/services/devops/). This gives you source code hosting, planning boards, building and testing infrastructure, and more. On top of that, you get [10 free parallel jobs](https://azure.microsoft.com/services/devops/pipelines/) for building your projects across all 3 major platforms: Windows, macOS and Linux. -After registering and creating your new project, simply add the following `build.yml` to the root of your extension's repository: +After registering and creating your new project, simply add the following `azure-pipelines.yml` to the root of your extension's repository. Other than the xvfb setup for Linux, the definition is straight-forward: ```yaml -jobs: - - job: Windows - pool: - name: Hosted VS2017 - demands: npm - steps: - - task: NodeTool@0 - displayName: 'Use Node 8.x' - inputs: - versionSpec: 8.x - - task: Npm@1 - displayName: 'Install dependencies' - inputs: - verbose: false - - task: Npm@1 - displayName: 'Compile sources' - inputs: - command: custom - verbose: false - customCommand: 'run compile' - - script: 'node node_modules/vscode/bin/test' - displayName: 'Run tests' - - job: macOS - pool: - name: Hosted macOS - demands: npm - steps: - - task: NodeTool@0 - displayName: 'Use Node 8.x' - inputs: - versionSpec: 8.x - - task: Npm@1 - displayName: 'Install dependencies' - inputs: - verbose: false - - task: Npm@1 - displayName: 'Compile sources' - inputs: - command: custom - verbose: false - customCommand: 'run compile' - - script: 'node node_modules/vscode/bin/test' - displayName: 'Run tests' - - job: Linux - pool: - name: Hosted Ubuntu 1604 - demands: npm - steps: - - task: NodeTool@0 - displayName: 'Use Node 8.x' - inputs: - versionSpec: 8.x - - task: Npm@1 - displayName: 'Install dependencies' - inputs: - verbose: false - - task: Npm@1 - displayName: 'Compile sources' - inputs: - command: custom - verbose: false - customCommand: 'run compile' - - script: | - set -e - /usr/bin/Xvfb :10 -ac >> /tmp/Xvfb.out 2>&1 & - disown -ar - displayName: 'Start xvfb' - - script: 'node node_modules/vscode/bin/test' - displayName: 'Run tests' - env: - DISPLAY: :10 +trigger: +- master + +strategy: + matrix: + linux: + imageName: 'ubuntu-16.04' + mac: + imageName: 'macos-10.13' + windows: + imageName: 'vs2017-win2016' + +pool: + vmImage: $(imageName) + +steps: + +- task: NodeTool@0 + inputs: + versionSpec: '8.x' + displayName: 'Install Node.js' + +- bash: | + if [ $AGENT_OS == "Linux" ]; then + set -e + /usr/bin/Xvfb :10 -ac >> /tmp/Xvfb.out 2>&1 & + disown -ar + echo "Started xvfb" + fi + displayName: Start xvfb + +- bash: | + yarn install + yarn compile + yarn test + displayName: Run Tests + env: + DISPLAY: :10 ``` -Next [create a new Pipeline](https://docs.microsoft.com/azure/devops/pipelines/get-started-yaml?view=vsts#get-your-first-build) in your DevOps project and point it to the `build.yml` file. Trigger a build and voilà: +Next [create a new Pipeline](https://docs.microsoft.com/azure/devops/pipelines/get-started-yaml?view=vsts#get-your-first-build) in your DevOps project and point it to the `azure-pipelines.yml` file. Trigger a build and voilà: ![pipelines](images/continuous-integration/pipelines.png) diff --git a/api/working-with-extensions/images/testing-extension/debug.mp4 b/api/working-with-extensions/images/testing-extension/debug.mp4 new file mode 100644 index 0000000000..d50b1f8c06 --- /dev/null +++ b/api/working-with-extensions/images/testing-extension/debug.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c0d2a0a11a60cd6db74a2e53fc657c2fcfddeef14589b579cccc65d90cca6ab +size 415650 diff --git a/api/working-with-extensions/images/testing-extension/launch-tests.png b/api/working-with-extensions/images/testing-extension/launch-tests.png deleted file mode 100644 index ec3254eedd..0000000000 --- a/api/working-with-extensions/images/testing-extension/launch-tests.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fa22285ae5068d61cc320e176bd54b1b61edc1832e356869c5914d13106687de -size 12229 diff --git a/api/working-with-extensions/images/testing-extension/pipelines-logo.png b/api/working-with-extensions/images/testing-extension/pipelines-logo.png deleted file mode 100644 index 6332624870..0000000000 --- a/api/working-with-extensions/images/testing-extension/pipelines-logo.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d56e0284ce613966b6a21c65040f67ba1ab9624d6c7b4ceee12a2d6e54b6dd5d -size 21292 diff --git a/api/working-with-extensions/images/testing-extension/pipelines.png b/api/working-with-extensions/images/testing-extension/pipelines.png deleted file mode 100644 index fa18d5e235..0000000000 --- a/api/working-with-extensions/images/testing-extension/pipelines.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:56ae4abd8cc6af8d755947227daf52f7d0a3bf2a8b75a36959f188d3397c2936 -size 286903 diff --git a/api/working-with-extensions/images/testing-extension/test-output.png b/api/working-with-extensions/images/testing-extension/test-output.png deleted file mode 100644 index 41b72dc5d3..0000000000 --- a/api/working-with-extensions/images/testing-extension/test-output.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f842aee46321d314c071340e4f794e34165cff790696f83ee0661381b64e18f8 -size 151411 diff --git a/api/working-with-extensions/testing-extension.md b/api/working-with-extensions/testing-extension.md index efb4fd8d1a..f662665ede 100644 --- a/api/working-with-extensions/testing-extension.md +++ b/api/working-with-extensions/testing-extension.md @@ -9,87 +9,197 @@ MetaDescription: Write tests for your Visual Studio Code extension (plug-in). # Testing Extension -Visual Studio Code supports running and debugging tests for your extension. These tests will run inside a special instance of VS Code named the `Extension Development Host`, and have full access to the VS Code API. We refer to these tests as integration tests, because they go beyond unit tests that can run without a VS Code instance. This documentation focuses on VS Code integration tests. For unit testing, you can use any popular testing framework, like [Mocha](https://mochajs.org/) or [Jasmine](https://jasmine.github.io/). +Visual Studio Code supports running and debugging tests for your extension. These tests will run inside a special instance of VS Code named the `Extension Development Host`, and have full access to the VS Code API. We refer to these tests as integration tests, because they go beyond unit tests that can run without a VS Code instance. This documentation focuses on VS Code integration tests. -## Yo Code test scaffolding +## Overview -If you are using the [yo code generator](https://github.com/Microsoft/vscode-generator-code), the generated projects include a sample test and instructions for running the tests. +If you are using the [Yeoman Generator](https://code.visualstudio.com/api/get-started/your-first-extension) to scaffold an extension, integration tests are already setup for you. This page assumes you have read through the [Getting Started](https://code.visualstudio.com/api/get-started/your-first-extension) section and explains how the integration tests work. -**Note**: The documentation below assumes that you created a TypeScript extension but the same also applies for a JavaScript extension. However, some file names may be different. +You can find the setup for this guide in [helloworld-test-sample](https://github.com/microsoft/vscode-extension-samples/tree/master/helloworld-test-sample). -After you've created a new extension and opened the project in VS Code, you can select the `Extension Tests` configuration from the drop-down at the top of the Debug View. +## Setup the test command -![launch tests](images/testing-extension/launch-tests.png) +VS Code provides two arguments to run extension tests. For example: -With this configuration chosen, when you run `Debug: Start` (`kb(workbench.action.debug.start)`), VS Code launches your extension in the `Extension Development Host` instance and runs your tests. Test output goes to the Debug Console where you can see the test results. +```bash +code --extensionDevelopmentPath=. --extensionTestsPath=./out/test +``` -![test output](images/testing-extension/test-output.png) +Although `code` is available on the CLI, it is not available in other environments such as a Continuous Integration machine. Besides, we might want to test against a different version of VS Code. Let us first setup a script to download VS Code and run the extension test using `vscode-test`: + +- `npm install -D vscode-test` +- Write a test script: + +```ts +import * as path from 'path'; + +import { runTests } from 'vscode-test'; + +async function go() { + try { + const extensionPath = path.resolve(__dirname, '../../'); + const testRunnerPath = path.resolve(__dirname, './testRunner.js'); + const testWorkspace = path.resolve(__dirname, '../../test/suite/fixture'); + + // Download VS Code, unzip it and run the integration test + await runTests({ + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + extensionPath, + // The path to test runner + // Passed to --extensionTestsPath + testRunnerPath, + // The workspace to open on starting up VS Code + testWorkspace + }); + } catch (err) { + console.error('Failed to run tests'); + process.exit(1); + } +} -The generated test uses the [Mocha test framework](https://mochajs.org/) for its test runner and library. +go(); +``` -The extension project comes with a `src/test` folder that includes an `index.ts` file which defines the Mocha test runner configuration and an `extension.test.ts` which has the example `Something 1` test. You can typically leave `index.ts` untouched, but you can modify it to adjust the configuration of Mocha. +The `runTests` API provides a lot of flexibility. For example, you can specify which version of VS Code you want to download, include additional launch arguments and do custom pre-test setup with path to the VS Code executable. You can read more about the API at [https://github.com/Microsoft/vscode-test](https://github.com/Microsoft/vscode-test). -``` -├── src -│ └── test -│ ├── extension.test.ts -│ └── index.ts -``` +## Setup the test runner -You can create more `test.ts` files under the `test` folder and they will automatically be built (to `out/test`) and run. The test runner will only consider files matching the name pattern `*.test.ts`. +When running the extension integration test, `--extensionTestsPath` points to a **test runner**. In this sample, we explain how to setup a test runner using [Mocha](https://mochajs.org/). -## Launch tests configuration +- `npm install -D mocha @types/mocha glob @types/glob` +- Include this test runner file: -The `Extension Tests` configuration is defined in the project's `.vscode\launch.json` file. It is similar the `Extension` configuration with the addition of the `--extensionTestsPath` argument which points to the compiled test files (assuming this is a TypeScript project). +```ts +import * as path from 'path'; +import * as Mocha from 'mocha'; +import * as glob from 'glob'; -```json -{ - "name": "Extension Tests", - "type": "extensionHost", - "request": "launch", - "runtimeExecutable": "${execPath}", - "args": [ - "--extensionDevelopmentPath=${workspaceFolder}", - "--extensionTestsPath=${workspaceFolder}/out/test" - ], - "outFiles": ["${workspaceFolder}/out/test/**/*.js"] +export function run(testsRoot: string, cb: (error: any, failures?: number) => void): void { + // Create the mocha test + const mocha = new Mocha({ + ui: 'tdd', + useColors: true + }); + + glob('**/*.test.js', { cwd: testsRoot }, (err, files) => { + if (err) { + return cb(err); + } + + // Add files to the test suite + files.forEach(f => mocha.addFile(path.resolve(testsRoot, f))); + + try { + // Run the mocha test + mocha.run(failures => { + cb(null, failures); + }); + } catch (err) { + cb(err); + } + }); } ``` -## Passing arguments to the Extension Development Host +Both the test runner and the `*.test.js` files have access to VS Code API. Here is a sample test: + +```ts +import * as assert from 'assert'; +import { after } from 'mocha'; + +// You can import and use all API from the 'vscode' module +// as well as import your extension to test it +import * as vscode from 'vscode'; +// import * as myExtension from '../extension'; + +suite('Extension Test Suite', () => { + + after(() => { + vscode.window.showInformationMessage('All tests done!'); + }); + + test('Sample test', () => { + assert.equal(-1, [1, 2, 3].indexOf(5)); + assert.equal(-1, [1, 2, 3].indexOf(0)); + }); +}); +``` + +## Debugging the tests -You can set the file or folder that the test instance should open by inserting the path at the front of the argument list for the launch configuration. +Debugging the tests is similar to debugging the extension. Here is a sample `launch.json`: ```json -"args": [ - "${workspaceFolder}/file or folder name", - "--extensionDevelopmentPath=${workspaceFolder}", - "--extensionTestsPath=${workspaceFolder}/out/test" -] +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Extension Tests", + "type": "extensionHost", + "request": "launch", + "runtimeExecutable": "${execPath}", + "args": [ + "--extensionDevelopmentPath=${workspaceFolder}", + "--extensionTestsPath=${workspaceFolder}/out/test/suite" + ], + "outFiles": ["${workspaceFolder}/out/test/**/*.js"], + } + ] +} ``` -This way you can run your tests with predictable content and folder structure. + + +## Tips -## Disabling other extensions +### Disabling other extensions while debugging -By default, the debug instance of VS Code will load any extension you've previously installed alongside the one you are developing. If you want to disable those extensions, add `"--disable-extensions"` to the argument list in the launch configuration. +When you debug an extension test in VS Code, VS Code uses the globally installed instance of VS Code and will load all installed extensions. You can add `--disable-extensions` configuration to the `launch.json` or the `additionalLaunchArgs` option of `vscode-test`'s `runTests` API. ```json -"args": [ - "--disable-extensions", - "--extensionDevelopmentPath=${workspaceFolder}", - "--extensionTestsPath=${workspaceFolder}/out/test" -] +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Extension Tests", + "type": "extensionHost", + "request": "launch", + "runtimeExecutable": "${execPath}", + "args": [ + "--disable-extensions", + "--extensionDevelopmentPath=${workspaceFolder}", + "--extensionTestsPath=${workspaceFolder}/out/test/suite" + ], + "outFiles": ["${workspaceFolder}/out/test/**/*.js"], + } + ] +} ``` -This will give large benefits to performance when running tests +```ts +await runTests({ + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + extensionPath, + // The path to test runner + // Passed to --extensionTestsPath + testRunnerPath, + // The workspace to open on starting up VS Code + testWorkspace, + additionalLaunchArgs: ['--disable-extensions'] +}); +``` -## Excluding test files from your extension package +### Custom setup with `vscode-test` -If you decide to share your extension, you may not want to include the tests in your extension package. The [`.vscodeignore`](/api/working-with-extensions/publishing-extension#advance-usage) file lets you exclude test files when you package and publish your extension with the [`vsce` publishing tool](/api/working-with-extensions/publishing-extension). By default, the `yo code` generated extension project excludes the `test` and `out/test` folders. +Sometimes you might want to run custom setups, such as running `code --install-extension` to install another extension before starting your test. `vscode-test` has a more granular API to accommodate that case: -``` -out/test/** +```ts +const vscodeExecutablePath = await downloadAndUnzipVSCode('1.34.0'); +child_process.spawnSync(vscodeExecutablePath, ['--install-extension', '']); ``` ## Next steps diff --git a/release-notes/v1_35.md b/release-notes/v1_35.md index 3c0183bd9c..37965ce8aa 100644 --- a/release-notes/v1_35.md +++ b/release-notes/v1_35.md @@ -24,6 +24,29 @@ We really appreciate people taking a look at our new features as soon as they ar If you find issues or have suggestions, you can enter them in the VS Code repository on [GitHub](https://github.com/Microsoft/vscode/issues). +## Extension authoring + +### Splitting `vscode` package into `@types/vscode` and vscode-test + +During the `event-stream` incident last year, we found that `vscode` package was affected as it pulls in 223 transitive dependencies totaling to 16MB. From time to time, these dependencies also cause GitHub security alerts for all extensions. Since then, we started slimming down `vscode`. + +`vscode` package today serves two purposes: +- Pull `vscode.d.ts` for extension development +- Run integration test by downloading and launching a local copy of VS Code + +So we splitted `vscode` into `@types/vscode` and `vscode-test`, two packages with more focused functionality. + +- [`@types/vscode`](https://www.npmjs.com/package/@types/vscode) contains `vscode.d.ts` for each release. For example, `npm i @types/vscode@1.34.0` installs VS Code 1.34 Extension API. +- [`vscode-test`](https://github.com/Microsoft/vscode-test) provides a set of API to run integration test with VS Code. You can learn more about using it in the [Testing Extensions](https://code.visualstudio.com/api/working-with-extensions/testing-extensionc) page. + +The old `vscode` package will continue to work, but new features will go to `vscode-test`. We suggest that you switch over to `vscode-test` which features slimmer dependency graph and a more flexible, explicitly documented API. + +Additionally: + +- [`vscode-dts`](https://github.com/microsoft/vscode-dts) allows you to to quickly download any version of VS Code API in CLI +- [`vsce`](https://github.com/Microsoft/vscode-vsce) checks `@types/vscode` version against `engines.vscode` to prevent you from using new API incompatible with old versions of VS Code +- [`helloworld-test-sample`](https://github.com/microsoft/vscode-extension-samples/tree/master/helloworld-test-sample) and the updated [Testing Extension](https://code.visualstudio.com/api/working-with-extensions/testing-extension) and [Continuous Integration](https://code.visualstudio.com/api/working-with-extensions/continuous-integration) documentation shows you how to use `vscode-test` + \ No newline at end of file From f096f1b1cde2e514617768eb8090477c028c7b13 Mon Sep 17 00:00:00 2001 From: Pine Wu Date: Fri, 24 May 2019 01:04:15 -0700 Subject: [PATCH 2/5] Update test doc to match sample --- api/working-with-extensions/testing-extension.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api/working-with-extensions/testing-extension.md b/api/working-with-extensions/testing-extension.md index f662665ede..1c579d12cc 100644 --- a/api/working-with-extensions/testing-extension.md +++ b/api/working-with-extensions/testing-extension.md @@ -39,7 +39,6 @@ async function go() { try { const extensionPath = path.resolve(__dirname, '../../'); const testRunnerPath = path.resolve(__dirname, './testRunner.js'); - const testWorkspace = path.resolve(__dirname, '../../test/suite/fixture'); // Download VS Code, unzip it and run the integration test await runTests({ @@ -48,9 +47,7 @@ async function go() { extensionPath, // The path to test runner // Passed to --extensionTestsPath - testRunnerPath, - // The workspace to open on starting up VS Code - testWorkspace + testRunnerPath }); } catch (err) { console.error('Failed to run tests'); From 0a7ec594107159127397d95b77bc15ed91bfb263 Mon Sep 17 00:00:00 2001 From: Pine Wu Date: Fri, 24 May 2019 01:59:15 -0700 Subject: [PATCH 3/5] Update release notes --- release-notes/v1_35.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release-notes/v1_35.md b/release-notes/v1_35.md index 37965ce8aa..aa81cb83c9 100644 --- a/release-notes/v1_35.md +++ b/release-notes/v1_35.md @@ -44,8 +44,9 @@ The old `vscode` package will continue to work, but new features will go to `vsc Additionally: - [`vscode-dts`](https://github.com/microsoft/vscode-dts) allows you to to quickly download any version of VS Code API in CLI -- [`vsce`](https://github.com/Microsoft/vscode-vsce) checks `@types/vscode` version against `engines.vscode` to prevent you from using new API incompatible with old versions of VS Code -- [`helloworld-test-sample`](https://github.com/microsoft/vscode-extension-samples/tree/master/helloworld-test-sample) and the updated [Testing Extension](https://code.visualstudio.com/api/working-with-extensions/testing-extension) and [Continuous Integration](https://code.visualstudio.com/api/working-with-extensions/continuous-integration) documentation shows you how to use `vscode-test` +- [`vsce`](https://github.com/Microsoft/vscode-vsce) now checks `@types/vscode` version against `engines.vscode` to prevent you from using new API incompatible with old versions of VS Code +- [`helloworld-test-sample`](https://github.com/microsoft/vscode-extension-samples/tree/master/helloworld-test-sample), [Testing Extension](https://code.visualstudio.com/api/working-with-extensions/testing-extension) page and [Continuous Integration](https://code.visualstudio.com/api/working-with-extensions/continuous-integration) page are updated to use `vscode-test` +- [VS Code Extension Generator](https://github.com/Microsoft/vscode-generator-code) now scaffolds extensions using `@types/vscode` and `vscode-test` From c719c697588cb3a86ab87d571dfa4183c26ffdfa Mon Sep 17 00:00:00 2001 From: Pine Wu Date: Fri, 24 May 2019 11:56:11 -0700 Subject: [PATCH 4/5] Address feedback --- .../continuous-integration.md | 19 ++++----- .../testing-extension.md | 41 +++++++++---------- release-notes/v1_35.md | 4 +- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/api/working-with-extensions/continuous-integration.md b/api/working-with-extensions/continuous-integration.md index 6a04900fd5..f374d8a8d4 100644 --- a/api/working-with-extensions/continuous-integration.md +++ b/api/working-with-extensions/continuous-integration.md @@ -9,7 +9,7 @@ MetaDescription: Use Continuous Integration for testing Visual Studio Code exten # Continuous Integration -Extension tests can be run on CI services. The `vscode-test` repository itself contains a sample extension that is tested on Azure Devops Pipelines. You can check out the [build pipeline](https://dev.azure.com/vscode/VSCode/_build?definitionId=14) or jump directly to the [build definition yaml file](https://github.com/microsoft/vscode-test/blob/master/azure-pipelines.yml). +Extension tests can be run on CI services. The `vscode-test` repository itself contains a sample extension that is tested on Azure Devops Pipelines. You can check out the [build pipeline](https://dev.azure.com/vscode/VSCode/_build?definitionId=14) or jump directly to the [build definition yaml file](https://github.com/microsoft/vscode-test/blob/master/sample/azure-pipelines.yml). ## Azure Pipelines @@ -43,18 +43,17 @@ steps: displayName: 'Install Node.js' - bash: | - if [ $AGENT_OS == "Linux" ]; then - set -e - /usr/bin/Xvfb :10 -ac >> /tmp/Xvfb.out 2>&1 & - disown -ar - echo "Started xvfb" - fi + set -e + /usr/bin/Xvfb :10 -ac >> /tmp/Xvfb.out 2>&1 & + disown -ar + echo "Started xvfb" displayName: Start xvfb + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - bash: | - yarn install - yarn compile - yarn test + # vscode-test has its extension located at /sample + cd sample + yarn && yarn compile && yarn test displayName: Run Tests env: DISPLAY: :10 diff --git a/api/working-with-extensions/testing-extension.md b/api/working-with-extensions/testing-extension.md index 1c579d12cc..7a554217ef 100644 --- a/api/working-with-extensions/testing-extension.md +++ b/api/working-with-extensions/testing-extension.md @@ -27,7 +27,7 @@ code --extensionDevelopmentPath=. --extensionTestsPath=./out/test Although `code` is available on the CLI, it is not available in other environments such as a Continuous Integration machine. Besides, we might want to test against a different version of VS Code. Let us first setup a script to download VS Code and run the extension test using `vscode-test`: -- `npm install -D vscode-test` +- `npm install --save-dev vscode-test` - Write a test script: ```ts @@ -35,27 +35,25 @@ import * as path from 'path'; import { runTests } from 'vscode-test'; -async function go() { - try { - const extensionPath = path.resolve(__dirname, '../../'); - const testRunnerPath = path.resolve(__dirname, './testRunner.js'); - - // Download VS Code, unzip it and run the integration test - await runTests({ - // The folder containing the Extension Manifest package.json - // Passed to `--extensionDevelopmentPath` - extensionPath, - // The path to test runner - // Passed to --extensionTestsPath - testRunnerPath - }); - } catch (err) { - console.error('Failed to run tests'); - process.exit(1); - } +async function main() { + try { + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + const extensionPath = path.resolve(__dirname, '../../'); + + // The path to test runner + // Passed to --extensionTestsPath + const testRunnerPath = path.resolve(__dirname, './suite'); + + // Download VS Code, unzip it and run the integration test + await runTests({ extensionPath, testRunnerPath }); + } catch (err) { + console.error('Failed to run tests'); + process.exit(1); + } } -go(); +main(); ``` The `runTests` API provides a lot of flexibility. For example, you can specify which version of VS Code you want to download, include additional launch arguments and do custom pre-test setup with path to the VS Code executable. You can read more about the API at [https://github.com/Microsoft/vscode-test](https://github.com/Microsoft/vscode-test). @@ -64,7 +62,7 @@ The `runTests` API provides a lot of flexibility. For example, you can specify w When running the extension integration test, `--extensionTestsPath` points to a **test runner**. In this sample, we explain how to setup a test runner using [Mocha](https://mochajs.org/). -- `npm install -D mocha @types/mocha glob @types/glob` +- `npm install --save-dev mocha @types/mocha glob @types/glob` - Include this test runner file: ```ts @@ -89,6 +87,7 @@ export function run(testsRoot: string, cb: (error: any, failures?: number) => vo try { // Run the mocha test + mocha.run(failures => cb(null, failures)); mocha.run(failures => { cb(null, failures); }); diff --git a/release-notes/v1_35.md b/release-notes/v1_35.md index aa81cb83c9..c496aad0c5 100644 --- a/release-notes/v1_35.md +++ b/release-notes/v1_35.md @@ -28,13 +28,13 @@ If you find issues or have suggestions, you can enter them in the VS Code reposi ### Splitting `vscode` package into `@types/vscode` and vscode-test -During the `event-stream` incident last year, we found that `vscode` package was affected as it pulls in 223 transitive dependencies totaling to 16MB. From time to time, these dependencies also cause GitHub security alerts for all extensions. Since then, we started slimming down `vscode`. +During the [`event-stream` incident](https://code.visualstudio.com/blogs/2018/11/26/event-stream) last year, we found that `vscode` package was affected as it pulls in 223 transitive dependencies totaling to 16MB. From time to time, these dependencies also cause GitHub security alerts for all extensions. Since then, we started slimming down `vscode`. `vscode` package today serves two purposes: - Pull `vscode.d.ts` for extension development - Run integration test by downloading and launching a local copy of VS Code -So we splitted `vscode` into `@types/vscode` and `vscode-test`, two packages with more focused functionality. +So we split `vscode` into `@types/vscode` and `vscode-test`, two packages with more focused functionality. - [`@types/vscode`](https://www.npmjs.com/package/@types/vscode) contains `vscode.d.ts` for each release. For example, `npm i @types/vscode@1.34.0` installs VS Code 1.34 Extension API. - [`vscode-test`](https://github.com/Microsoft/vscode-test) provides a set of API to run integration test with VS Code. You can learn more about using it in the [Testing Extensions](https://code.visualstudio.com/api/working-with-extensions/testing-extensionc) page. From ecc908dd69d9c8e115d52b959a3e21c0e0a7794b Mon Sep 17 00:00:00 2001 From: Pine Wu Date: Tue, 28 May 2019 09:45:25 +0800 Subject: [PATCH 5/5] Address feedback --- .../testing-extension.md | 109 ++++++++++-------- 1 file changed, 60 insertions(+), 49 deletions(-) diff --git a/api/working-with-extensions/testing-extension.md b/api/working-with-extensions/testing-extension.md index 7a554217ef..062dc4b397 100644 --- a/api/working-with-extensions/testing-extension.md +++ b/api/working-with-extensions/testing-extension.md @@ -13,22 +13,35 @@ Visual Studio Code supports running and debugging tests for your extension. Thes ## Overview -If you are using the [Yeoman Generator](https://code.visualstudio.com/api/get-started/your-first-extension) to scaffold an extension, integration tests are already setup for you. This page assumes you have read through the [Getting Started](https://code.visualstudio.com/api/get-started/your-first-extension) section and explains how the integration tests work. +If you are using the [Yeoman Generator](https://code.visualstudio.com/api/get-started/your-first-extension) to scaffold an extension, integration tests are already setup for you. -You can find the setup for this guide in [helloworld-test-sample](https://github.com/microsoft/vscode-extension-samples/tree/master/helloworld-test-sample). +In the generated extension, you can use `npm run test` or `yarn test` to run the integration tests that: -## Setup the test command +- Downloads and unzips latest version of VS Code +- Runs the mocha tests specified by the test runner -VS Code provides two arguments to run extension tests. For example: +Alternatively, you can find the setup for this guide in [helloworld-test-sample](https://github.com/microsoft/vscode-extension-samples/tree/master/helloworld-test-sample). The rest of this document explains these files in the context of the sample: + +- The **test script** ([`src/test/runTest.ts`](https://github.com/microsoft/vscode-extension-samples/blob/master/helloworld-test-sample/src/test/runTest.ts)) +- The **test runner** ([`src/test/suite/index.ts`](https://github.com/microsoft/vscode-extension-samples/blob/master/helloworld-test-sample/src/test/suite/index.ts)) + +## The test script + +VS Code provides two CLI parameters for running extension tests. For example: ```bash -code --extensionDevelopmentPath=. --extensionTestsPath=./out/test +# - Launches VS Code Extension Host +# - Loads the extension at +# - Executes the test runner at +code \ +--extensionDevelopmentPath= \ +--extensionTestsPath= ``` -Although `code` is available on the CLI, it is not available in other environments such as a Continuous Integration machine. Besides, we might want to test against a different version of VS Code. Let us first setup a script to download VS Code and run the extension test using `vscode-test`: - -- `npm install --save-dev vscode-test` -- Write a test script: +The **test script** ([`src/test/runTest.ts`](https://github.com/microsoft/vscode-extension-samples/blob/master/helloworld-test-sample/src/test/runTest.ts)) uses the `vscode-test` API to simplify the process of downloading, unzipping and launching VS Code with extension test parameters. The `vscode-test` API also allows: +- Launching VS Code with a specific workspace +- Downloading a different version of VS Code than the latest +- Launching VS Code with additional CLI parameters ```ts import * as path from 'path'; @@ -36,34 +49,31 @@ import * as path from 'path'; import { runTests } from 'vscode-test'; async function main() { - try { - // The folder containing the Extension Manifest package.json - // Passed to `--extensionDevelopmentPath` - const extensionPath = path.resolve(__dirname, '../../'); - - // The path to test runner - // Passed to --extensionTestsPath - const testRunnerPath = path.resolve(__dirname, './suite'); - - // Download VS Code, unzip it and run the integration test - await runTests({ extensionPath, testRunnerPath }); - } catch (err) { - console.error('Failed to run tests'); - process.exit(1); - } + try { + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + const extensionPath = path.resolve(__dirname, '../../'); + + // The path to test runner + // Passed to --extensionTestsPath + const testRunnerPath = path.resolve(__dirname, './suite'); + + // Download VS Code, unzip it and run the integration test + await runTests({ extensionPath, testRunnerPath }); + } catch (err) { + console.error('Failed to run tests'); + process.exit(1); + } } main(); ``` -The `runTests` API provides a lot of flexibility. For example, you can specify which version of VS Code you want to download, include additional launch arguments and do custom pre-test setup with path to the VS Code executable. You can read more about the API at [https://github.com/Microsoft/vscode-test](https://github.com/Microsoft/vscode-test). - -## Setup the test runner +You can find more API usage examples at [microsoft/vscode-test](https://github.com/microsoft/vscode-test). -When running the extension integration test, `--extensionTestsPath` points to a **test runner**. In this sample, we explain how to setup a test runner using [Mocha](https://mochajs.org/). +## The test runner -- `npm install --save-dev mocha @types/mocha glob @types/glob` -- Include this test runner file: +When running the extension integration test, `--extensionTestsPath` points to the **test runner** ([`src/test/suite/index.ts`](https://github.com/microsoft/vscode-extension-samples/blob/master/helloworld-test-sample/src/test/suite/index.ts)) that programatically runs the test suite. ```ts import * as path from 'path'; @@ -73,9 +83,10 @@ import * as glob from 'glob'; export function run(testsRoot: string, cb: (error: any, failures?: number) => void): void { // Create the mocha test const mocha = new Mocha({ - ui: 'tdd', - useColors: true + ui: 'tdd' }); + // Use any mocha API + mocha.useColors(true); glob('**/*.test.js', { cwd: testsRoot }, (err, files) => { if (err) { @@ -88,9 +99,6 @@ export function run(testsRoot: string, cb: (error: any, failures?: number) => vo try { // Run the mocha test mocha.run(failures => cb(null, failures)); - mocha.run(failures => { - cb(null, failures); - }); } catch (err) { cb(err); } @@ -98,7 +106,7 @@ export function run(testsRoot: string, cb: (error: any, failures?: number) => vo } ``` -Both the test runner and the `*.test.js` files have access to VS Code API. Here is a sample test: +Both the test runner and the `*.test.js` files have access to VS Code API. Here is a the sample test ([src/test/suite/extension.test.ts](https://github.com/microsoft/vscode-extension-samples/blob/master/helloworld-test-sample/src/test/suite/extension.test.ts)): ```ts import * as assert from 'assert'; @@ -111,14 +119,14 @@ import * as vscode from 'vscode'; suite('Extension Test Suite', () => { - after(() => { - vscode.window.showInformationMessage('All tests done!'); - }); + after(() => { + vscode.window.showInformationMessage('All tests done!'); + }); - test('Sample test', () => { - assert.equal(-1, [1, 2, 3].indexOf(5)); - assert.equal(-1, [1, 2, 3].indexOf(0)); - }); + test('Sample test', () => { + assert.equal(-1, [1, 2, 3].indexOf(5)); + assert.equal(-1, [1, 2, 3].indexOf(0)); + }); }); ``` @@ -177,14 +185,10 @@ When you debug an extension test in VS Code, VS Code uses the globally installed ```ts await runTests({ - // The folder containing the Extension Manifest package.json - // Passed to `--extensionDevelopmentPath` extensionPath, - // The path to test runner - // Passed to --extensionTestsPath testRunnerPath, - // The workspace to open on starting up VS Code - testWorkspace, + // Additional CLI parameters for launching `code`. + // Use `code --help` to find all CLI parameters additionalLaunchArgs: ['--disable-extensions'] }); ``` @@ -195,7 +199,14 @@ Sometimes you might want to run custom setups, such as running `code --install-e ```ts const vscodeExecutablePath = await downloadAndUnzipVSCode('1.34.0'); +// Custom setup child_process.spawnSync(vscodeExecutablePath, ['--install-extension', '']); +await runTests({ + // Use the specified `code` executable instead of downloading + vscodeExecutablePath, + extensionPath, + testRunnerPath +}) ``` ## Next steps