Async textmate tokenization

.vscode/tasks.json

@@ -262,15 +262,7 @@
 			"type": "shell",
 			"command": "node_modules/.bin/http-server --cors --port 5001 -a 127.0.0.1 -s -c-1",
 			"isBackground": true,
-			"problemMatcher": {
-				"pattern": {
-					"regexp": ""
-				},
-				"background": {
-					"beginsPattern": ".*",
-					"endsPattern": ".*"
-				}
-			},
+			"problemMatcher": [],
 			"dependsOn": [
 				"Core - Build"
 			]

package.json

@@ -86,7 +86,7 @@
     "vscode-oniguruma": "1.7.0",
     "vscode-proxy-agent": "^0.12.0",
     "vscode-regexpp": "^3.1.0",
-    "vscode-textmate": "8.0.0",
+    "vscode-textmate": "9.0.0",
     "xterm": "5.2.0-beta.29",
     "xterm-addon-canvas": "0.4.0-beta.7",
     "xterm-addon-search": "0.11.0",
@@ -240,4 +240,4 @@
     "elliptic": "^6.5.3",
     "nwmatcher": "^1.4.4"
   }
-}
+}

remote/package.json

@@ -23,7 +23,7 @@
     "vscode-oniguruma": "1.7.0",
     "vscode-proxy-agent": "^0.12.0",
     "vscode-regexpp": "^3.1.0",
-    "vscode-textmate": "8.0.0",
+    "vscode-textmate": "9.0.0",
     "xterm": "5.2.0-beta.29",
     "xterm-addon-canvas": "0.4.0-beta.7",
     "xterm-addon-search": "0.11.0",

remote/web/package.json

@@ -10,7 +10,7 @@
 		"jschardet": "3.0.0",
 		"tas-client-umd": "0.1.6",
 		"vscode-oniguruma": "1.7.0",
-		"vscode-textmate": "8.0.0",
+		"vscode-textmate": "9.0.0",
 		"xterm": "5.2.0-beta.29",
 		"xterm-addon-canvas": "0.4.0-beta.7",
 		"xterm-addon-search": "0.11.0",

remote/web/yarn.lock

@@ -63,10 +63,10 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz#439bfad8fe71abd7798338d1cd3dc53a8beea94b"
   integrity sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==
 
-vscode-textmate@8.0.0:
-  version "8.0.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d"
-  integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==
+vscode-textmate@9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c"
+  integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg==
 
 [email protected]:
   version "0.4.0-beta.7"

remote/yarn.lock

@@ -810,10 +810,10 @@ vscode-regexpp@^3.1.0:
   resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f"
   integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg==
 
-vscode-textmate@8.0.0:
-  version "8.0.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d"
-  integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==
+vscode-textmate@9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c"
+  integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg==
 
 vscode-windows-ca-certs@^0.3.0:
   version "0.3.0"

src/buildfile.js

@@ -54,6 +54,7 @@ exports.workerProfileAnalysis = [createEditorWorkerModuleDescription('vs/platfor
 
 exports.workbenchDesktop = [
 	createEditorWorkerModuleDescription('vs/workbench/contrib/output/common/outputLinkComputer'),
+	createEditorWorkerModuleDescription('vs/workbench/services/textMate/browser/worker/textMate.worker'),
 	createModuleDescription('vs/workbench/contrib/debug/node/telemetryApp'),
 	createModuleDescription('vs/platform/files/node/watcher/watcherMain'),
 	createModuleDescription('vs/platform/terminal/node/ptyHostMain'),
@@ -62,6 +63,7 @@ exports.workbenchDesktop = [
 
 exports.workbenchWeb = [
 	createEditorWorkerModuleDescription('vs/workbench/contrib/output/common/outputLinkComputer'),
+	createEditorWorkerModuleDescription('vs/workbench/services/textMate/browser/worker/textMate.worker'),
 	createModuleDescription('vs/code/browser/workbench/workbench', ['vs/workbench/workbench.web.main'])
 ];
 

src/vs/editor/common/config/editorConfigurationSchema.ts

@@ -95,6 +95,11 @@ const editorConfiguration: IConfigurationNode = {
 			default: 20_000,
 			description: nls.localize('maxTokenizationLineLength', "Lines above this length will not be tokenized for performance reasons")
 		},
+		'editor.experimental.asyncTokenization': {
+			type: 'boolean',
+			default: false,
+			description: nls.localize('editor.experimental.asyncTokenization', "Controls whether the tokenization should happen asynchronously on a web worker.")
+		},
 		'editor.language.brackets': {
 			type: ['array', 'null'],
 			default: null, // We want to distinguish the empty array from not configured.

src/vs/editor/common/model/textModelTokens.ts

@@ -30,7 +30,7 @@ const enum Constants {
  * An array that avoids being sparse by always
  * filling up unused indices with a default value.
  */
-class ContiguousGrowingArray<T> {
+export class ContiguousGrowingArray<T> {
 
 	private _store: T[] = [];
 

src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts

@@ -21,7 +21,7 @@ import { FontStyle, ColorId, StandardTokenType, TokenMetadata } from 'vs/editor/
 import { ILanguageService } from 'vs/editor/common/languages/language';
 import { INotificationService } from 'vs/platform/notification/common/notification';
 import { findMatchingThemeRule } from 'vs/workbench/services/textMate/common/TMHelper';
-import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import type { IGrammar, IToken, StateStack } from 'vscode-textmate';
 import { IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { CancellationTokenSource } from 'vs/base/common/cancellation';
@@ -43,7 +43,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib
 	}
 
 	private _editor: ICodeEditor;
-	private _textMateService: ITextMateService;
+	private _textMateService: ITextMateTokenizationService;
 	private _themeService: IWorkbenchThemeService;
 	private _languageService: ILanguageService;
 	private _notificationService: INotificationService;
@@ -53,7 +53,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib
 
 	constructor(
 		editor: ICodeEditor,
-		@ITextMateService textMateService: ITextMateService,
+		@ITextMateTokenizationService textMateService: ITextMateTokenizationService,
 		@ILanguageService languageService: ILanguageService,
 		@IWorkbenchThemeService themeService: IWorkbenchThemeService,
 		@INotificationService notificationService: INotificationService,
@@ -187,7 +187,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
 	private readonly _editor: IActiveCodeEditor;
 	private readonly _languageService: ILanguageService;
 	private readonly _themeService: IWorkbenchThemeService;
-	private readonly _textMateService: ITextMateService;
+	private readonly _textMateService: ITextMateTokenizationService;
 	private readonly _notificationService: INotificationService;
 	private readonly _configurationService: IConfigurationService;
 	private readonly _languageFeaturesService: ILanguageFeaturesService;
@@ -197,7 +197,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
 
 	constructor(
 		editor: IActiveCodeEditor,
-		textMateService: ITextMateService,
+		textMateService: ITextMateTokenizationService,
 		languageService: ILanguageService,
 		themeService: IWorkbenchThemeService,
 		notificationService: INotificationService,

src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts

@@ -14,7 +14,7 @@ import { ILanguageService } from 'vs/editor/common/languages/language';
 import { Extensions, IJSONContributionRegistry } from 'vs/platform/jsonschemas/common/jsonContributionRegistry';
 import { Registry } from 'vs/platform/registry/common/platform';
 import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions';
-import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { getParseErrorMessage } from 'vs/base/common/jsonErrorMessages';
 import { IExtensionResourceLoaderService } from 'vs/platform/extensionResourceLoader/common/extensionResourceLoader';
 import { hash } from 'vs/base/common/hash';
@@ -95,7 +95,7 @@ export class LanguageConfigurationFileHandler extends Disposable {
 	private readonly _done = new Map<string, number>();
 
 	constructor(
-		@ITextMateService textMateService: ITextMateService,
+		@ITextMateTokenizationService textMateService: ITextMateTokenizationService,
 		@ILanguageService private readonly _languageService: ILanguageService,
 		@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
 		@IExtensionService private readonly _extensionService: IExtensionService,

src/vs/workbench/contrib/codeEditor/electron-sandbox/startDebugTextMate.ts

@@ -7,7 +7,7 @@ import * as nls from 'vs/nls';
 import { Range } from 'vs/editor/common/core/range';
 import { Action2, registerAction2 } from 'vs/platform/actions/common/actions';
 import { Categories } from 'vs/platform/action/common/actionCommonCategories';
-import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { IModelService } from 'vs/editor/common/services/model';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { URI } from 'vs/base/common/uri';
@@ -52,7 +52,7 @@ class StartDebugTextMate extends Action2 {
 	}
 
 	async run(accessor: ServicesAccessor) {
-		const textMateService = accessor.get(ITextMateService);
+		const textMateService = accessor.get(ITextMateTokenizationService);
 		const modelService = accessor.get(IModelService);
 		const editorService = accessor.get(IEditorService);
 		const codeEditorService = accessor.get(ICodeEditorService);

src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts

@@ -10,7 +10,7 @@ import { IInstantiationService, ServicesAccessor } from 'vs/platform/instantiati
 import { IWorkbenchThemeService, IWorkbenchColorTheme } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { EditorResourceAccessor } from 'vs/workbench/common/editor';
-import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import type { IGrammar, StateStack } from 'vscode-textmate';
 import { TokenizationRegistry } from 'vs/editor/common/languages';
 import { TokenMetadata } from 'vs/editor/common/encodedTokenAttributes';
@@ -92,7 +92,7 @@ class Snapper {
 	constructor(
 		@ILanguageService private readonly languageService: ILanguageService,
 		@IWorkbenchThemeService private readonly themeService: IWorkbenchThemeService,
-		@ITextMateService private readonly textMateService: ITextMateService
+		@ITextMateTokenizationService private readonly textMateService: ITextMateTokenizationService
 	) {
 	}