Fix hang in constraint solver for cyclic TypeVar lower bounds (#11413) (#11415)

* Fix hang in constraint solver for cyclic TypeVar lower bounds (#11413)

When the constraint solver established a TypeVar's first lower bound, it
would unconditionally record the source type even when that source type
contained the destination TypeVar nested inside another generic
constructor (e.g. R := R | Awaitable[R], as produced when matching wrapt
2.1.x's wrap_function_wrapper against a generic make_wrapper). Each
subsequent round of solveAndApplyConstraints substituted R with its own
ever-growing bound, producing exponentially larger types and hanging the
analyzer.

Add an occurs check in assignUnconstrainedTypeVar: if adjSrcType
references destType at a strictly nested position, refuse the assignment
rather than recording a non-finitely-solvable constraint. Top-level
union members that are exactly the TypeVar (e.g. T := T | int from
protocol matching against T | int) are explicitly allowed and resolved
by the existing logic.

Adds a regression test (paramSpec56) that hangs without the fix and
completes in ~640ms with it.

* Apply prettier formatting

Author: rchiodo

packages/pyright-internal/src/analyzer/constraintSolver.ts

@@ -763,6 +763,25 @@ function assignUnconstrainedTypeVar(
     } else {
         if (!curLowerBound || isTypeSame(destType, curLowerBound)) {
             // There was previously no lower bound. We've now established one.
+            // Apply an occurs check: if `adjSrcType` references `destType`
+            // (the TypeVar being solved) at a strictly nested position
+            // (e.g. `R := F[R]` or `R := R | Awaitable[R]`), recording it as
+            // the lower bound creates a cyclic constraint that subsequent
+            // widening / substitution rounds expand into an exponentially
+            // growing recursive type. Such a constraint has no finite
+            // solution, so report the assignment as a failure rather than
+            // letting the analyzer hang. See microsoft/pyright#11413.
+            //
+            // Top-level union members that are exactly `destType` (e.g.
+            // `T := T | int` arising from protocol matching against `T | int`)
+            // are *not* considered cyclic - the original `adjSrcType` is
+            // recorded as the lower bound and existing logic resolves it.
+            if (typeVarOccursIn(destType, adjSrcType)) {
+                diag?.addMessage(
+                    LocAddendum.typeAssignmentMismatch().format(evaluator.printSrcDestTypes(adjSrcType, destType))
+                );
+                return false;
+            }
             newLowerBound = adjSrcType;
         } else if (isTypeSame(curLowerBound, adjSrcType, {}, recursionCount)) {
             // If this is an invariant context and there is currently no upper bound
@@ -1289,6 +1308,39 @@ function assignParamSpec(
     return isAssignable;
 }
 
+// Returns true if `typeVar` appears strictly inside `type` (i.e. nested
+// within another type, not as the top-level type itself or as a top-level
+// subtype of a union). Used as an occurs check to detect cyclic constraints
+// during widening. A bare top-level reference is fine (`T := T` is the
+// identity); a top-level union member can be subtracted before solving;
+// only a strictly nested reference (`T := F[T]`) has no finite solution.
+function typeVarOccursIn(typeVar: TypeVarType, type: Type): boolean {
+    // A bare top-level TypeVar reference is not a cycle. Compare by name +
+    // scope id rather than identity since pyright sometimes clones TypeVars.
+    if (isTypeVar(type) && type.shared.name === typeVar.shared.name && type.priv.scopeId === typeVar.priv.scopeId) {
+        return false;
+    }
+
+    // Top-level union members that are exactly `typeVar` are also fine; only
+    // count occurrences strictly inside other types.
+    if (isUnion(type)) {
+        for (const subtype of type.priv.subtypes) {
+            if (typeVarOccursIn(typeVar, subtype)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    const tvars = getTypeVarArgsRecursive(type);
+    for (const tv of tvars) {
+        if (tv.shared.name === typeVar.shared.name && tv.priv.scopeId === typeVar.priv.scopeId) {
+            return true;
+        }
+    }
+    return false;
+}
+
 // For normal TypeVars, the constraint solver can widen a type by combining
 // two otherwise incompatible types into a union. For TypeVarTuples, we need
 // to do the equivalent operation for unpacked tuples.

packages/pyright-internal/src/tests/samples/paramSpec56.py

@@ -0,0 +1,58 @@
+# This sample tests a case derived from the wrapt 2.1.x stubs that
+# previously caused pyright to hang. A function parameter is typed
+# as a union of three generic Callable aliases, each parameterized
+# with the same ParamSpec and TypeVar, and the argument is itself a
+# generic Callable whose return type is a union (`R | Awaitable[R]`).
+# Bidirectional inference must terminate (and not blow up
+# combinatorially) on this case.
+
+from collections.abc import Awaitable, Callable
+from typing import Any, ParamSpec, TypeVar
+
+P = ParamSpec("P")
+R = TypeVar("R", covariant=True)
+
+GenericCallableWrapperFunction = Callable[
+    [Callable[P, R], Any, tuple[Any, ...], dict[str, Any]], R
+]
+ClassMethodWrapperFunction = Callable[
+    [type[Any], Callable[P, R], Any, tuple[Any, ...], dict[str, Any]], R
+]
+InstanceMethodWrapperFunction = Callable[
+    [Any, Callable[P, R], Any, tuple[Any, ...], dict[str, Any]], R
+]
+WrapperFunction = (
+    GenericCallableWrapperFunction[P, R]
+    | ClassMethodWrapperFunction[P, R]
+    | InstanceMethodWrapperFunction[P, R]
+)
+
+
+def wrap_function_wrapper(
+    target: str, name: str, wrapper: WrapperFunction[P, R]
+) -> None: ...
+
+
+def make_async_wrapper[**P1, R1]() -> Callable[
+    [Callable[P1, R1], Any, tuple[Any, ...], dict[str, Any]], Awaitable[R1]
+]: ...
+
+
+def make_sync_wrapper[**P1, R1]() -> Callable[
+    [Callable[P1, R1], Any, tuple[Any, ...], dict[str, Any]], R1
+]: ...
+
+
+def make_wrapper[**P1, R1](
+    is_async: bool,
+) -> Callable[
+    [Callable[P1, R1], Any, tuple[Any, ...], dict[str, Any]], R1 | Awaitable[R1]
+]:
+    return make_async_wrapper() if is_async else make_sync_wrapper()
+
+
+# This call must terminate. The constraint solver sees a cyclic
+# constraint (R := R | Awaitable[R]) which has no finite solution,
+# so it refuses to bind R and the call returns no useful inferred type.
+# Analysis must not hang.
+wrap_function_wrapper("example", "target", make_wrapper(False))

packages/pyright-internal/src/tests/typeEvaluator4.test.ts

@@ -883,6 +883,18 @@ test('ParamSpec55', () => {
     TestUtils.validateResults(results, 1);
 });
 
+// Regression test for a hang reported in microsoft/pyright#11413,
+// reproduced from the wrapt 2.1.x stubs. Analysis must terminate;
+// the call below sets up a cyclic constraint (R := R | Awaitable[R])
+// which has no finite solution. The constraint solver detects the cycle
+// and refuses to bind R, but the failure isn't surfaced as a user-
+// visible error here (it occurs during bidirectional inference for an
+// argument expression). The important thing is that analysis completes.
+test('ParamSpec56', () => {
+    const results = TestUtils.typeAnalyzeSampleFiles(['paramSpec56.py']);
+    TestUtils.validateResults(results, 0);
+});
+
 test('Slice1', () => {
     const results = TestUtils.typeAnalyzeSampleFiles(['slice1.py']);
     TestUtils.validateResults(results, 0);