Refactors tokenization to support custom background tokenizer.

src/vs/editor/common/languages.ts

@@ -19,6 +19,7 @@ import { Selection } from 'vs/editor/common/core/selection';
 import { LanguageId } from 'vs/editor/common/encodedTokenAttributes';
 import * as model from 'vs/editor/common/model';
 import { TokenizationRegistry as TokenizationRegistryImpl } from 'vs/editor/common/tokenizationRegistry';
+import { ContiguousMultilineTokens } from 'vs/editor/common/tokens/contiguousMultilineTokens';
 import { ExtensionIdentifier } from 'vs/platform/extensions/common/extensions';
 import { IMarkerData } from 'vs/platform/markers/common/markers';
 
@@ -33,14 +34,11 @@ export interface ILanguageIdCodec {
 export class Token {
 	_tokenBrand: void = undefined;
 
-	public readonly offset: number;
-	public readonly type: string;
-	public readonly language: string;
-
-	constructor(offset: number, type: string, language: string) {
-		this.offset = offset;
-		this.type = type;
-		this.language = language;
+	constructor(
+		public readonly offset: number,
+		public readonly type: string,
+		public readonly language: string,
+	) {
 	}
 
 	public toString(): string {
@@ -54,12 +52,10 @@ export class Token {
 export class TokenizationResult {
 	_tokenizationResultBrand: void = undefined;
 
-	public readonly tokens: Token[];
-	public readonly endState: IState;
-
-	constructor(tokens: Token[], endState: IState) {
-		this.tokens = tokens;
-		this.endState = endState;
+	constructor(
+		public readonly tokens: Token[],
+		public readonly endState: IState,
+	) {
 	}
 }
 
@@ -69,21 +65,30 @@ export class TokenizationResult {
 export class EncodedTokenizationResult {
 	_encodedTokenizationResultBrand: void = undefined;
 
+	constructor(
+		/**
+		 * The tokens in binary format. Each token occupies two array indices. For token i:
+		 *  - at offset 2*i => startIndex
+		 *  - at offset 2*i + 1 => metadata
+		 *
+		 */
+		public readonly tokens: Uint32Array,
+		public readonly endState: IState,
+	) {
+	}
+}
+
+export interface IBackgroundTokenizer extends IDisposable {
 	/**
-	 * The tokens in binary format. Each token occupies two array indices. For token i:
-	 *  - at offset 2*i => startIndex
-	 *  - at offset 2*i + 1 => metadata
+	 * Instructs the background tokenizer to set the tokens for the given range again.
 	 *
+	 * This might be necessary if the renderer overwrote those tokens with heuristically computed ones for some viewport,
+	 * when the change does not even propagate to that viewport.
 	 */
-	public readonly tokens: Uint32Array;
-	public readonly endState: IState;
-
-	constructor(tokens: Uint32Array, endState: IState) {
-		this.tokens = tokens;
-		this.endState = endState;
-	}
+	requestTokens(startLineNumber: number, endLineNumberExclusive: number): void;
 }
 
+
 /**
  * @internal
  */
@@ -94,6 +99,26 @@ export interface ITokenizationSupport {
 	tokenize(line: string, hasEOL: boolean, state: IState): TokenizationResult;
 
 	tokenizeEncoded(line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult;
+
+	/**
+	 * Can be/return undefined if default background tokenization should be used.
+	 */
+	createBackgroundTokenizer?(textModel: model.ITextModel, store: IBackgroundTokenizationStore): IBackgroundTokenizer | undefined;
+}
+
+/**
+ * @internal
+ */
+export interface IBackgroundTokenizationStore {
+	setTokens(tokens: ContiguousMultilineTokens[]): void;
+
+	setEndState(lineNumber: number, state: IState): void;
+
+	/**
+	 * Should be called to indicate that the background tokenization has finished for now.
+	 * (This triggers bracket pair colorization to re-parse the bracket pairs with token information)
+	 */
+	backgroundTokenizationFinished(): void;
 }
 
 /**

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree.ts

@@ -56,8 +56,7 @@ export class BracketPairsTree extends Disposable {
 	) {
 		super();
 
-		if (textModel.tokenization.backgroundTokenizationState === BackgroundTokenizationState.Uninitialized) {
-			// There are no token information yet
+		if (!textModel.tokenization.hasTokens) {
 			const brackets = this.brackets.getSingleLanguageBracketTokens(this.textModel.getLanguageId());
 			const tokenizer = new FastTokenizer(this.textModel.getValue(), brackets);
 			this.initialAstWithoutTokens = parseDocument(tokenizer, [], undefined, true);
@@ -67,7 +66,8 @@ export class BracketPairsTree extends Disposable {
 			// Directly create the tree with token information.
 			this.initialAstWithoutTokens = undefined;
 			this.astWithTokens = this.parseDocumentFromTextBuffer([], undefined, false);
-		} else if (textModel.tokenization.backgroundTokenizationState === BackgroundTokenizationState.InProgress) {
+		} else {
+			// We missed some token changes already, so we cannot use the fast tokenizer + delta increments
 			this.initialAstWithoutTokens = this.parseDocumentFromTextBuffer([], undefined, true);
 			this.astWithTokens = this.initialAstWithoutTokens;
 		}
@@ -103,6 +103,7 @@ export class BracketPairsTree extends Disposable {
 	}
 
 	public handleContentChanged(change: IModelContentChangedEvent) {
+		// Must be sorted in ascending order
 		const edits = change.changes.map(c => {
 			const range = Range.lift(c.range);
 			return new TextEditInfo(