Removed some circular module dependencies. (#10485)

erictraut · web-flow · commit 321b6bf687c6 · 2025-05-21T14:37:20.000-07:00
diff --git a/packages/pyright-internal/src/analyzer/parseTreeUtils.ts b/packages/pyright-internal/src/analyzer/parseTreeUtils.ts
@@ -46,7 +46,7 @@ import {
 } from '../parser/parseNodes';
 import { OperatorTypeNameMap, ParseNodeTypeNameMap } from '../parser/parseNodeUtils';
 import { ParseFileResults } from '../parser/parser';
-import { TokenizerOutput } from '../parser/tokenizer';
+import { Tokenizer, TokenizerOutput } from '../parser/tokenizer';
 import { KeywordType, OperatorType, StringToken, StringTokenFlags, Token, TokenType } from '../parser/tokenizerTypes';
 import { getScope } from './analyzerNodeInfo';
 import { ParseTreeWalker, getChildNodes } from './parseTreeWalker';
@@ -1864,7 +1864,7 @@ export function getTokenIndexAtLeft(
             continue;
         }
 
-        if (!includeWhitespace && isWhitespace(token)) {
+        if (!includeWhitespace && Tokenizer.isWhitespace(token)) {
             continue;
         }
 
@@ -1919,10 +1919,6 @@ export function getTokenAfter(tokens: TextRangeCollection<Token>, position: numb
     return tokens.getItemAt(index);
 }
 
-export function isWhitespace(token: Token) {
-    return token.type === TokenType.NewLine || token.type === TokenType.Indent || token.type === TokenType.Dedent;
-}
-
 export function getTokenAtIndex(tokens: TextRangeCollection<Token>, index: number) {
     if (index < 0) {
         return undefined;
@@ -2382,7 +2378,7 @@ export function getFullStatementRange(
 
 export function isBlankLine(tokenizerOutput: TokenizerOutput, text: string, line: number) {
     const span = tokenizerOutput.lines.getItemAt(line);
-    return containsOnlyWhitespace(text, span);
+    return containsOnlyWhitespace(text, span.start, TextRange.getEnd(span));
 }
 
 export function isUnannotatedFunction(node: FunctionNode) {
@@ -2700,7 +2696,7 @@ export function getPreviousNonWhitespaceToken(tokens: TextRangeCollection<Token>
 
     while (tokenIndex >= 0) {
         const token = tokens.getItemAt(tokenIndex);
-        if (!isWhitespace(token)) {
+        if (!Tokenizer.isWhitespace(token)) {
             return token;
         }
 
@@ -2711,7 +2707,7 @@ export function getPreviousNonWhitespaceToken(tokens: TextRangeCollection<Token>
 }
 
 export function getNextNonWhitespaceToken(tokens: TextRangeCollection<Token>, offset: number): Token | undefined {
-    return getNextMatchingToken(tokens, offset, (token) => !isWhitespace(token));
+    return getNextMatchingToken(tokens, offset, (token) => !Tokenizer.isWhitespace(token));
 }
 
 export function getNextMatchingToken(
diff --git a/packages/pyright-internal/src/analyzer/sourceFile.ts b/packages/pyright-internal/src/analyzer/sourceFile.ts
@@ -677,7 +677,7 @@ export class SourceFile {
                     configOptions,
                     this._uri,
                     fileContents!,
-                    this._ipythonMode,
+                    this._ipythonMode !== IPythonMode.None,
                     diagSink
                 );
 
@@ -1448,15 +1448,15 @@ export class SourceFile {
         configOptions: ConfigOptions,
         fileUri: Uri,
         fileContents: string,
-        ipythonMode: IPythonMode,
+        useNotebookMode: boolean,
         diagSink: DiagnosticSink
     ): ParseFileResults {
         // Use the configuration options to determine the environment zin which
         // this source file will be executed.
         const execEnvironment = configOptions.findExecEnvironment(fileUri);
 
         const parseOptions = new ParseOptions();
-        parseOptions.ipythonMode = ipythonMode;
+        parseOptions.useNotebookMode = useNotebookMode;
         if (fileUri.pathEndsWith('pyi')) {
             parseOptions.isStubFile = true;
         }
diff --git a/packages/pyright-internal/src/common/core.ts b/packages/pyright-internal/src/common/core.ts
@@ -6,8 +6,6 @@
  * Various helpers that don't have a dependency on other code files.
  */
 
-import { TextRange } from './textRange';
-
 export const enum Comparison {
     LessThan = -1,
     EqualTo = 0,
@@ -179,9 +177,9 @@ export function getEnumNames<T>(enumType: T) {
     return result;
 }
 
-export function containsOnlyWhitespace(text: string, span?: TextRange) {
-    if (span) {
-        text = text.substring(span.start, TextRange.getEnd(span));
+export function containsOnlyWhitespace(text: string, start?: number, end?: number) {
+    if (start !== undefined) {
+        text = text.substring(start, end);
     }
 
     return /^\s*$/.test(text);
diff --git a/packages/pyright-internal/src/common/docRange.ts b/packages/pyright-internal/src/common/docRange.ts
@@ -0,0 +1,15 @@
+/*
+ * docRange.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ *
+ * Specifies the range of text within a document.
+ */
+
+import { Range } from './textRange';
+import { Uri } from './uri/uri';
+
+export interface DocumentRange {
+    uri: Uri;
+    range: Range;
+}
diff --git a/packages/pyright-internal/src/common/textRange.ts b/packages/pyright-internal/src/common/textRange.ts
@@ -7,8 +7,6 @@
  * Specifies the range of text within a larger string.
  */
 
-import { Uri } from './uri/uri';
-
 export interface TextRange {
     readonly start: number;
     readonly length: number;
@@ -132,11 +130,6 @@ export namespace Range {
 }
 
 // Represents a range within a particular document.
-export interface DocumentRange {
-    uri: Uri;
-    range: Range;
-}
-
 export function comparePositions(a: Position, b: Position) {
     if (a.line < b.line) {
         return -1;
diff --git a/packages/pyright-internal/src/languageServerBase.ts b/packages/pyright-internal/src/languageServerBase.ts
@@ -94,6 +94,7 @@ import { ConsoleInterface, ConsoleWithLogLevel, LogLevel } from './common/consol
 import { Diagnostic as AnalyzerDiagnostic, DiagnosticCategory } from './common/diagnostic';
 import { DiagnosticRule } from './common/diagnosticRules';
 import { FileDiagnostics } from './common/diagnosticSink';
+import { DocumentRange } from './common/docRange';
 import { FileSystem, ReadOnlyFileSystem } from './common/fileSystem';
 import { FileWatcherEventType } from './common/fileWatcher';
 import { Host } from './common/host';
@@ -107,7 +108,7 @@ import { fromLSPAny, isNullProgressReporter } from './common/lspUtils';
 import { ProgressReportTracker, ProgressReporter } from './common/progressReporter';
 import { ServiceKeys } from './common/serviceKeys';
 import { ServiceProvider } from './common/serviceProvider';
-import { DocumentRange, Position, Range } from './common/textRange';
+import { Position, Range } from './common/textRange';
 import { Uri } from './common/uri/uri';
 import { convertUriToLspUriString } from './common/uri/uriUtils';
 import { AnalyzerServiceExecutor } from './languageService/analyzerServiceExecutor';
diff --git a/packages/pyright-internal/src/languageService/definitionProvider.ts b/packages/pyright-internal/src/languageService/definitionProvider.ts
@@ -28,11 +28,12 @@ import { OverloadedType, TypeCategory, isOverloaded } from '../analyzer/types';
 import { throwIfCancellationRequested } from '../common/cancellationUtils';
 import { appendArray } from '../common/collectionUtils';
 import { isDefined } from '../common/core';
+import { DocumentRange } from '../common/docRange';
 import { ProgramView } from '../common/extensibility';
 import { convertOffsetsToRange, convertPositionToOffset } from '../common/positionUtils';
 import { ServiceKeys } from '../common/serviceKeys';
 import { ServiceProvider } from '../common/serviceProvider';
-import { DocumentRange, Position, rangesAreEqual } from '../common/textRange';
+import { Position, rangesAreEqual } from '../common/textRange';
 import { Uri } from '../common/uri/uri';
 import { ParseNode, ParseNodeType } from '../parser/parseNodes';
 import { ParseFileResults } from '../parser/parser';
diff --git a/packages/pyright-internal/src/languageService/navigationUtils.ts b/packages/pyright-internal/src/languageService/navigationUtils.ts
@@ -6,8 +6,8 @@
  * Helper functions for navigating files.
  */
 import { Location } from 'vscode-languageserver-types';
+import { DocumentRange } from '../common/docRange';
 import { ReadOnlyFileSystem } from '../common/fileSystem';
-import { DocumentRange } from '../common/textRange';
 import { Uri } from '../common/uri/uri';
 import { convertUriToLspUriString } from '../common/uri/uriUtils';
 
diff --git a/packages/pyright-internal/src/languageService/referencesProvider.ts b/packages/pyright-internal/src/languageService/referencesProvider.ts
@@ -22,11 +22,12 @@ import { throwIfCancellationRequested } from '../common/cancellationUtils';
 import { appendArray } from '../common/collectionUtils';
 import { isDefined } from '../common/core';
 import { assertNever } from '../common/debug';
+import { DocumentRange } from '../common/docRange';
 import { ProgramView, ReferenceUseCase, SymbolUsageProvider } from '../common/extensibility';
 import { ReadOnlyFileSystem } from '../common/fileSystem';
 import { convertOffsetToPosition, convertPositionToOffset } from '../common/positionUtils';
 import { ServiceKeys } from '../common/serviceKeys';
-import { DocumentRange, Position, Range, TextRange, doesRangeContain } from '../common/textRange';
+import { Position, Range, TextRange, doesRangeContain } from '../common/textRange';
 import { Uri } from '../common/uri/uri';
 import { NameNode, ParseNode, ParseNodeType } from '../parser/parseNodes';
 import { ParseFileResults } from '../parser/parser';
diff --git a/packages/pyright-internal/src/parser/parser.ts b/packages/pyright-internal/src/parser/parser.ts
@@ -11,7 +11,6 @@
  * into an abstract syntax tree (AST).
  */
 
-import { IPythonMode } from '../analyzer/sourceFile';
 import { appendArray } from '../common/collectionUtils';
 import { assert } from '../common/debug';
 import { Diagnostic, DiagnosticAddendum } from '../common/diagnostic';
@@ -166,15 +165,15 @@ export class ParseOptions {
     pythonVersion: PythonVersion;
     reportInvalidStringEscapeSequence: boolean;
     skipFunctionAndClassBody: boolean;
-    ipythonMode: IPythonMode;
+    useNotebookMode: boolean;
     reportErrorsForParsedStringContents: boolean;
 
     constructor() {
         this.isStubFile = false;
         this.pythonVersion = latestStablePythonVersion;
         this.reportInvalidStringEscapeSequence = false;
         this.skipFunctionAndClassBody = false;
-        this.ipythonMode = IPythonMode.None;
+        this.useNotebookMode = false;
         this.reportErrorsForParsedStringContents = false;
     }
 }
@@ -399,7 +398,7 @@ export class Parser {
             textOffset,
             textLength,
             initialParenDepth,
-            this._parseOptions.ipythonMode
+            this._parseOptions.useNotebookMode
         );
         this._tokenIndex = 0;
     }
diff --git a/packages/pyright-internal/src/parser/tokenizer.ts b/packages/pyright-internal/src/parser/tokenizer.ts
@@ -10,8 +10,6 @@
  * Converts a Python program text stream into a stream of tokens.
  */
 
-import { isWhitespace } from '../analyzer/parseTreeUtils';
-import { IPythonMode } from '../analyzer/sourceFile';
 import { Char } from '../common/charCodes';
 import { cloneStr } from '../common/core';
 import { TextRange } from '../common/textRange';
@@ -244,15 +242,15 @@ export class Tokenizer {
     private _singleQuoteCount = 0;
     private _doubleQuoteCount = 0;
 
-    // ipython mode
-    private _ipythonMode = IPythonMode.None;
+    // Assume Jupyter notebook tokenization rules?
+    private _useNotebookMode = false;
 
     tokenize(
         text: string,
         start?: number,
         length?: number,
         initialParenDepth = 0,
-        ipythonMode = IPythonMode.None
+        useNotebookMode = false
     ): TokenizerOutput {
         if (start === undefined) {
             start = 0;
@@ -275,7 +273,7 @@ export class Tokenizer {
         this._parenDepth = initialParenDepth;
         this._lineRanges = [];
         this._indentAmounts = [];
-        this._ipythonMode = ipythonMode;
+        this._useNotebookMode = useNotebookMode;
 
         const end = start + length;
 
@@ -375,6 +373,10 @@ export class Tokenizer {
         return _operatorInfo[operatorType];
     }
 
+    static isWhitespace(token: Token) {
+        return token.type === TokenType.NewLine || token.type === TokenType.Indent || token.type === TokenType.Dedent;
+    }
+
     static isPythonKeyword(name: string, includeSoftKeywords = false): boolean {
         const keyword = _keywords.get(name);
         if (!keyword) {
@@ -460,7 +462,7 @@ export class Tokenizer {
             return true;
         }
 
-        if (this._ipythonMode) {
+        if (this._useNotebookMode) {
             const kind = this._getIPythonMagicsKind();
             if (kind === 'line') {
                 this._handleIPythonMagics(
@@ -1243,7 +1245,7 @@ export class Tokenizer {
         }
 
         const prevToken = this._tokens.length > 0 ? this._tokens[this._tokens.length - 1] : undefined;
-        if (prevToken !== undefined && !isWhitespace(prevToken)) {
+        if (prevToken !== undefined && !Tokenizer.isWhitespace(prevToken)) {
             return undefined;
         }
 
diff --git a/packages/pyright-internal/src/tests/harness/fourslash/testState.ts b/packages/pyright-internal/src/tests/harness/fourslash/testState.ts
@@ -41,16 +41,18 @@ import { ConsoleInterface, ConsoleWithLogLevel, NullConsole } from '../../../com
 import { Comparison, isNumber, isString } from '../../../common/core';
 import * as debug from '../../../common/debug';
 import { DiagnosticCategory } from '../../../common/diagnostic';
+import { DocumentRange } from '../../../common/docRange';
 import { PyrightDocStringService } from '../../../common/docStringService';
 import { FileEditAction } from '../../../common/editAction';
 import { ReadOnlyFileSystem } from '../../../common/fileSystem';
+import { Host } from '../../../common/host';
 import { LanguageServerInterface } from '../../../common/languageServerInterface';
 import { getFileExtension, normalizePath, normalizeSlashes } from '../../../common/pathUtils';
 import { convertOffsetToPosition, convertPositionToOffset } from '../../../common/positionUtils';
 import { ServiceProvider } from '../../../common/serviceProvider';
 import { createServiceProvider } from '../../../common/serviceProviderExtensions';
 import { compareStringsCaseInsensitive, compareStringsCaseSensitive } from '../../../common/stringUtils';
-import { DocumentRange, Position, Range as PositionRange, TextRange, rangesAreEqual } from '../../../common/textRange';
+import { Position, Range as PositionRange, TextRange, rangesAreEqual } from '../../../common/textRange';
 import { TextRangeCollection } from '../../../common/textRangeCollection';
 import { Uri } from '../../../common/uri/uri';
 import { UriEx, getFileSpec } from '../../../common/uri/uriUtils';
@@ -72,6 +74,7 @@ import { SignatureHelpProvider } from '../../../languageService/signatureHelpPro
 import { ParseNode } from '../../../parser/parseNodes';
 import { ParseFileResults } from '../../../parser/parser';
 import { Tokenizer } from '../../../parser/tokenizer';
+import { PartialStubService } from '../../../partialStubService';
 import { PyrightFileSystem } from '../../../pyrightFileSystem';
 import { NormalWorkspace, WellKnownWorkspaceKinds, Workspace, createInitStatus } from '../../../workspaceFactory';
 import { TestAccessHost } from '../testAccessHost';
@@ -98,8 +101,6 @@ import {
     getRangeByMarkerName,
 } from './testStateUtils';
 import { verifyWorkspaceEdit } from './workspaceEditTestUtils';
-import { Host } from '../../../common/host';
-import { PartialStubService } from '../../../partialStubService';
 
 export interface TextChange {
     span: TextRange;
diff --git a/packages/pyright-internal/src/tests/lsp/languageServerTestUtils.ts b/packages/pyright-internal/src/tests/lsp/languageServerTestUtils.ts
@@ -58,7 +58,6 @@ import {
     WorkDoneProgressCreateRequest,
 } from 'vscode-languageserver/node';
 import { PythonPathResult } from '../../analyzer/pythonPathUtils';
-import { IPythonMode } from '../../analyzer/sourceFile';
 import { deserialize } from '../../backgroundThreadBase';
 import { PythonPlatform } from '../../common/configOptions';
 import { toBoolean } from '../../common/core';
@@ -245,10 +244,10 @@ export function updateSettingsMap(info: PyrightServerInfo, settings: { item: Con
     settingsMap.set(info, [...settings, ...current]);
 }
 
-export function getParseResults(fileContents: string, isStubFile = false, ipythonMode: IPythonMode = 0) {
+export function getParseResults(fileContents: string, isStubFile = false, useNotebookMode = false) {
     const diagSink = new DiagnosticSink();
     const parseOptions = new ParseOptions();
-    parseOptions.ipythonMode = ipythonMode;
+    parseOptions.useNotebookMode = useNotebookMode;
     parseOptions.isStubFile = isStubFile;
     parseOptions.pythonVersion = pythonVersion3_10;
     parseOptions.skipFunctionAndClassBody = false;
diff --git a/packages/pyright-internal/src/tests/positionUtils.test.ts b/packages/pyright-internal/src/tests/positionUtils.test.ts
@@ -51,7 +51,7 @@ test('getLineEndOffset with mixed style ending', () => {
     verifyLineEnding(code, 2, 5);
 });
 
-test('End of file position and offest conversion', () => {
+test('End of file position and offset conversion', () => {
     const code = 'hello\n';
 
     const t = new Tokenizer();
