Don't cache Ternary.Maybe results when recursion is encountered during variance measurement

src/compiler/checker.ts

@@ -16407,6 +16407,7 @@ namespace ts {
             let lastSkippedInfo: [Type, Type] | undefined;
             let incompatibleStack: [DiagnosticMessage, (string | number)?, (string | number)?, (string | number)?, (string | number)?][] = [];
             let inPropertyCheck = false;
+            let recursiveVariances = false;
 
             Debug.assert(relation !== identityRelation || !errorNode, "no error reporting in identity checking");
 
@@ -17255,9 +17256,11 @@ namespace ts {
                 depth--;
                 if (result) {
                     if (result === Ternary.True || depth === 0) {
-                        // If result is definitely true, record all maybe keys as having succeeded
-                        for (let i = maybeStart; i < maybeCount; i++) {
-                            relation.set(maybeKeys[i], RelationComparisonResult.Succeeded | propagatingVarianceFlags);
+                        if (result === Ternary.True || !recursiveVariances) {
+                            // If result is definitely true, record all maybe keys as having succeeded
+                            for (let i = maybeStart; i < maybeCount; i++) {
+                                relation.set(maybeKeys[i], RelationComparisonResult.Succeeded | propagatingVarianceFlags);
+                            }
                         }
                         maybeCount = maybeStart;
                     }
@@ -17327,6 +17330,7 @@ namespace ts {
                     !(source.aliasTypeArgumentsContainsMarker || target.aliasTypeArgumentsContainsMarker)) {
                     const variances = getAliasVariances(source.aliasSymbol);
                     if (variances === emptyArray) {
+                        recursiveVariances = true;
                         return Ternary.Maybe;
                     }
                     const varianceResult = relateVariances(source.aliasTypeArguments, target.aliasTypeArguments, variances, intersectionState);
@@ -17600,6 +17604,7 @@ namespace ts {
                         // effectively means we measure variance only from type parameter occurrences that aren't nested in
                         // recursive instantiations of the generic type.
                         if (variances === emptyArray) {
+                            recursiveVariances = true;
                             return Ternary.Maybe;
                         }
                         const varianceResult = relateVariances(getTypeArguments(<TypeReference>source), getTypeArguments(<TypeReference>target), variances, intersectionState);

tests/baselines/reference/varianceMeasurement.errors.txt

@@ -22,9 +22,11 @@ tests/cases/compiler/varianceMeasurement.ts(57,7): error TS2322: Type 'Fn<string
   Type 'unknown' is not assignable to type 'string'.
 tests/cases/compiler/varianceMeasurement.ts(62,7): error TS2322: Type 'Fn<string, number>' is not assignable to type 'Fn<string, 0>'.
   Type 'number' is not assignable to type '0'.
+tests/cases/compiler/varianceMeasurement.ts(75,7): error TS2322: Type 'C<unknown, number>' is not assignable to type 'C<unknown, string>'.
+  Type 'number' is not assignable to type 'string'.
 
 
-==== tests/cases/compiler/varianceMeasurement.ts (8 errors) ====
+==== tests/cases/compiler/varianceMeasurement.ts (9 errors) ====
     // The type below should be invariant in T but is measured as covariant because
     // we don't analyze recursive references.
 
@@ -119,4 +121,20 @@ tests/cases/compiler/varianceMeasurement.ts(62,7): error TS2322: Type 'Fn<string
           ~~~
 !!! error TS2322: Type 'Fn<string, number>' is not assignable to type 'Fn<string, 0>'.
 !!! error TS2322:   Type 'number' is not assignable to type '0'.
+
+    // Repro from #39947
+
+    interface I<Dummy, V> {
+      c: C<Dummy, V>;
+    }
+
+    class C<Dummy, V> {
+      declare sub: I<Dummy, V>;
+      declare covariance: V;
+    }
+
+    const c1: C<unknown, string> = new C<unknown, number>();  // Error
+          ~~
+!!! error TS2322: Type 'C<unknown, number>' is not assignable to type 'C<unknown, string>'.
+!!! error TS2322:   Type 'number' is not assignable to type 'string'.
 

tests/baselines/reference/varianceMeasurement.js

@@ -61,6 +61,19 @@ const fn2: Fn<'a', number> = fn;
 // Covariant in B
 const fn3: Fn<string, unknown> = fn;
 const fn4: Fn<string, 0> = fn;  // Error
+
+// Repro from #39947
+
+interface I<Dummy, V> {
+  c: C<Dummy, V>;
+}
+
+class C<Dummy, V> {
+  declare sub: I<Dummy, V>;
+  declare covariance: V;
+}
+
+const c1: C<unknown, string> = new C<unknown, number>();  // Error
 
 
 //// [varianceMeasurement.js]
@@ -81,3 +94,9 @@ var fn2 = fn;
 // Covariant in B
 var fn3 = fn;
 var fn4 = fn; // Error
+var C = /** @class */ (function () {
+    function C() {
+    }
+    return C;
+}());
+var c1 = new C(); // Error

tests/baselines/reference/varianceMeasurement.symbols

@@ -183,3 +183,38 @@ const fn4: Fn<string, 0> = fn;  // Error
 >Fn : Symbol(Fn, Decl(varianceMeasurement.ts, 44, 31))
 >fn : Symbol(fn, Decl(varianceMeasurement.ts, 53, 13))
 
+// Repro from #39947
+
+interface I<Dummy, V> {
+>I : Symbol(I, Decl(varianceMeasurement.ts, 61, 30))
+>Dummy : Symbol(Dummy, Decl(varianceMeasurement.ts, 65, 12))
+>V : Symbol(V, Decl(varianceMeasurement.ts, 65, 18))
+
+  c: C<Dummy, V>;
+>c : Symbol(I.c, Decl(varianceMeasurement.ts, 65, 23))
+>C : Symbol(C, Decl(varianceMeasurement.ts, 67, 1))
+>Dummy : Symbol(Dummy, Decl(varianceMeasurement.ts, 65, 12))
+>V : Symbol(V, Decl(varianceMeasurement.ts, 65, 18))
+}
+
+class C<Dummy, V> {
+>C : Symbol(C, Decl(varianceMeasurement.ts, 67, 1))
+>Dummy : Symbol(Dummy, Decl(varianceMeasurement.ts, 69, 8))
+>V : Symbol(V, Decl(varianceMeasurement.ts, 69, 14))
+
+  declare sub: I<Dummy, V>;
+>sub : Symbol(C.sub, Decl(varianceMeasurement.ts, 69, 19))
+>I : Symbol(I, Decl(varianceMeasurement.ts, 61, 30))
+>Dummy : Symbol(Dummy, Decl(varianceMeasurement.ts, 69, 8))
+>V : Symbol(V, Decl(varianceMeasurement.ts, 69, 14))
+
+  declare covariance: V;
+>covariance : Symbol(C.covariance, Decl(varianceMeasurement.ts, 70, 27))
+>V : Symbol(V, Decl(varianceMeasurement.ts, 69, 14))
+}
+
+const c1: C<unknown, string> = new C<unknown, number>();  // Error
+>c1 : Symbol(c1, Decl(varianceMeasurement.ts, 74, 5))
+>C : Symbol(C, Decl(varianceMeasurement.ts, 67, 1))
+>C : Symbol(C, Decl(varianceMeasurement.ts, 67, 1))
+

tests/baselines/reference/varianceMeasurement.types

@@ -131,3 +131,25 @@ const fn4: Fn<string, 0> = fn;  // Error
 >fn4 : Fn<string, 0>
 >fn : Fn<string, number>
 
+// Repro from #39947
+
+interface I<Dummy, V> {
+  c: C<Dummy, V>;
+>c : C<Dummy, V>
+}
+
+class C<Dummy, V> {
+>C : C<Dummy, V>
+
+  declare sub: I<Dummy, V>;
+>sub : I<Dummy, V>
+
+  declare covariance: V;
+>covariance : V
+}
+
+const c1: C<unknown, string> = new C<unknown, number>();  // Error
+>c1 : C<unknown, string>
+>new C<unknown, number>() : C<unknown, number>
+>C : typeof C
+

tests/cases/compiler/varianceMeasurement.ts

@@ -62,3 +62,16 @@ const fn2: Fn<'a', number> = fn;
 // Covariant in B
 const fn3: Fn<string, unknown> = fn;
 const fn4: Fn<string, 0> = fn;  // Error
+
+// Repro from #39947
+
+interface I<Dummy, V> {
+  c: C<Dummy, V>;
+}
+
+class C<Dummy, V> {
+  declare sub: I<Dummy, V>;
+  declare covariance: V;
+}
+
+const c1: C<unknown, string> = new C<unknown, number>();  // Error