Async textmate tokenization

src/vs/editor/common/config/editorConfigurationSchema.ts

@@ -95,6 +95,11 @@ const editorConfiguration: IConfigurationNode = {
 			default: 20_000,
 			description: nls.localize('maxTokenizationLineLength', "Lines above this length will not be tokenized for performance reasons")
 		},
+		'editor.experimental.asyncTokenization': {
+			type: 'boolean',
+			default: false,
+			description: nls.localize('editor.experimental.asyncTokenization', "Controls whether the tokenization should happen asynchronously on a web worker.")
+		},
 		'editor.language.brackets': {
 			type: ['array', 'null'],
 			default: null, // We want to distinguish the empty array from not configured.

src/vs/editor/common/model/textModelTokens.ts

@@ -30,7 +30,7 @@ const enum Constants {
  * An array that avoids being sparse by always
  * filling up unused indices with a default value.
  */
-class ContiguousGrowingArray<T> {
+export class ContiguousGrowingArray<T> {
 
 	private _store: T[] = [];
 

src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts

@@ -21,7 +21,7 @@ import { FontStyle, ColorId, StandardTokenType, TokenMetadata } from 'vs/editor/
 import { ILanguageService } from 'vs/editor/common/languages/language';
 import { INotificationService } from 'vs/platform/notification/common/notification';
 import { findMatchingThemeRule } from 'vs/workbench/services/textMate/common/TMHelper';
-import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate';
+import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import type { IGrammar, IToken, StateStack } from 'vscode-textmate';
 import { IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { CancellationTokenSource } from 'vs/base/common/cancellation';
@@ -43,7 +43,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib
 	}
 
 	private _editor: ICodeEditor;
-	private _textMateService: ITextMateService;
+	private _textMateService: ITextMateTokenizationFeature;
 	private _themeService: IWorkbenchThemeService;
 	private _languageService: ILanguageService;
 	private _notificationService: INotificationService;
@@ -53,7 +53,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib
 
 	constructor(
 		editor: ICodeEditor,
-		@ITextMateService textMateService: ITextMateService,
+		@ITextMateTokenizationFeature textMateService: ITextMateTokenizationFeature,
 		@ILanguageService languageService: ILanguageService,
 		@IWorkbenchThemeService themeService: IWorkbenchThemeService,
 		@INotificationService notificationService: INotificationService,
@@ -187,7 +187,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
 	private readonly _editor: IActiveCodeEditor;
 	private readonly _languageService: ILanguageService;
 	private readonly _themeService: IWorkbenchThemeService;
-	private readonly _textMateService: ITextMateService;
+	private readonly _textMateService: ITextMateTokenizationFeature;
 	private readonly _notificationService: INotificationService;
 	private readonly _configurationService: IConfigurationService;
 	private readonly _languageFeaturesService: ILanguageFeaturesService;
@@ -197,7 +197,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
 
 	constructor(
 		editor: IActiveCodeEditor,
-		textMateService: ITextMateService,
+		textMateService: ITextMateTokenizationFeature,
 		languageService: ILanguageService,
 		themeService: IWorkbenchThemeService,
 		notificationService: INotificationService,

src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts

@@ -14,7 +14,7 @@ import { ILanguageService } from 'vs/editor/common/languages/language';
 import { Extensions, IJSONContributionRegistry } from 'vs/platform/jsonschemas/common/jsonContributionRegistry';
 import { Registry } from 'vs/platform/registry/common/platform';
 import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions';
-import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate';
+import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { getParseErrorMessage } from 'vs/base/common/jsonErrorMessages';
 import { IExtensionResourceLoaderService } from 'vs/platform/extensionResourceLoader/common/extensionResourceLoader';
 import { hash } from 'vs/base/common/hash';
@@ -95,7 +95,7 @@ export class LanguageConfigurationFileHandler extends Disposable {
 	private readonly _done = new Map<string, number>();
 
 	constructor(
-		@ITextMateService textMateService: ITextMateService,
+		@ITextMateTokenizationFeature textMateService: ITextMateTokenizationFeature,
 		@ILanguageService private readonly _languageService: ILanguageService,
 		@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
 		@IExtensionService private readonly _extensionService: IExtensionService,

src/vs/workbench/contrib/codeEditor/electron-sandbox/startDebugTextMate.ts

@@ -7,7 +7,7 @@ import * as nls from 'vs/nls';
 import { Range } from 'vs/editor/common/core/range';
 import { Action2, registerAction2 } from 'vs/platform/actions/common/actions';
 import { Categories } from 'vs/platform/action/common/actionCommonCategories';
-import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate';
+import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { IModelService } from 'vs/editor/common/services/model';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { URI } from 'vs/base/common/uri';
@@ -52,7 +52,7 @@ class StartDebugTextMate extends Action2 {
 	}
 
 	async run(accessor: ServicesAccessor) {
-		const textMateService = accessor.get(ITextMateService);
+		const textMateService = accessor.get(ITextMateTokenizationFeature);
 		const modelService = accessor.get(IModelService);
 		const editorService = accessor.get(IEditorService);
 		const codeEditorService = accessor.get(ICodeEditorService);

src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts

@@ -10,7 +10,7 @@ import { IInstantiationService, ServicesAccessor } from 'vs/platform/instantiati
 import { IWorkbenchThemeService, IWorkbenchColorTheme } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { EditorResourceAccessor } from 'vs/workbench/common/editor';
-import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate';
+import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import type { IGrammar, StateStack } from 'vscode-textmate';
 import { TokenizationRegistry } from 'vs/editor/common/languages';
 import { TokenMetadata } from 'vs/editor/common/encodedTokenAttributes';
@@ -92,7 +92,7 @@ class Snapper {
 	constructor(
 		@ILanguageService private readonly languageService: ILanguageService,
 		@IWorkbenchThemeService private readonly themeService: IWorkbenchThemeService,
-		@ITextMateService private readonly textMateService: ITextMateService
+		@ITextMateTokenizationFeature private readonly textMateService: ITextMateTokenizationFeature
 	) {
 	}