Fix stack overflow in exhaustiveness due to recursive HIR opaque hidden types

compiler/rustc_pattern_analysis/src/rustc.rs

@@ -1,5 +1,6 @@
 use std::fmt;
 use std::iter::once;
+use std::ops::ControlFlow;
 
 use rustc_abi::{FIRST_VARIANT, FieldIdx, Integer, VariantIdx};
 use rustc_arena::DroplessArena;
@@ -11,7 +12,8 @@ use rustc_middle::mir::{self, Const};
 use rustc_middle::thir::{self, Pat, PatKind, PatRange, PatRangeBoundary};
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{
-    self, FieldDef, OpaqueTypeKey, ScalarInt, Ty, TyCtxt, TypeVisitableExt, VariantDef,
+    self, FieldDef, OpaqueTypeKey, ScalarInt, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable,
+    TypeVisitableExt, TypeVisitor, VariantDef,
 };
 use rustc_middle::{bug, span_bug};
 use rustc_session::lint;
@@ -135,11 +137,22 @@ impl<'p, 'tcx: 'p> RustcPatCtxt<'p, 'tcx> {
     /// Returns the hidden type corresponding to this key if the body under analysis is allowed to
     /// know it.
     fn reveal_opaque_key(&self, key: OpaqueTypeKey<'tcx>) -> Option<Ty<'tcx>> {
-        self.typeck_results
-            .concrete_opaque_types
-            .get(&key.def_id)
-            .map(|x| ty::EarlyBinder::bind(x.ty).instantiate(self.tcx, key.args))
+        if let Some(hidden_ty) = self.typeck_results.concrete_opaque_types.get(&key.def_id) {
+            let ty = ty::EarlyBinder::bind(hidden_ty.ty).instantiate(self.tcx, key.args);
+            if ty.visit_with(&mut RecursiveOpaque { def_id: key.def_id.into() }).is_continue() {
+                Some(ty)
+            } else {
+                // HACK: We skip revealing opaque types which recursively expand
+                // to themselves. This is because we may infer hidden types like
+                // `Opaque<T> = Opaque<Opaque<T>>` or `Opaque<T> = Opaque<(T,)>`
+                // in hir typeck.
+                None
+            }
+        } else {
+            None
+        }
     }
+
     // This can take a non-revealed `Ty` because it reveals opaques itself.
     pub fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
         !ty.inhabited_predicate(self.tcx).apply_revealing_opaque(
@@ -1105,3 +1118,20 @@ pub fn analyze_match<'p, 'tcx>(
 
     Ok(report)
 }
+
+struct RecursiveOpaque {
+    def_id: DefId,
+}
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for RecursiveOpaque {
+    type Result = ControlFlow<()>;
+
+    fn visit_ty(&mut self, t: Ty<'tcx>) -> Self::Result {
+        if let ty::Alias(ty::Opaque, alias_ty) = t.kind() {
+            if alias_ty.def_id == self.def_id {
+                return ControlFlow::Break(());
+            }
+        }
+
+        if t.has_opaque_types() { t.super_visit_with(self) } else { ControlFlow::Continue(()) }
+    }
+}

tests/ui/impl-trait/recursive-in-exhaustiveness.current.stderr

@@ -0,0 +1,56 @@
+warning: function cannot return without recursing
+  --> $DIR/recursive-in-exhaustiveness.rs:17:1
+   |
+LL | fn build<T>(x: T) -> impl Sized {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cannot return without recursing
+LL |
+LL |     let (x,) = (build(x),);
+   |                 -------- recursive call site
+   |
+   = help: a `loop` may express intention better if this is on purpose
+   = note: `#[warn(unconditional_recursion)]` on by default
+
+warning: function cannot return without recursing
+  --> $DIR/recursive-in-exhaustiveness.rs:27:1
+   |
+LL | fn build2<T>(x: T) -> impl Sized {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cannot return without recursing
+...
+LL |     let (x,) = (build2(x),);
+   |                 --------- recursive call site
+   |
+   = help: a `loop` may express intention better if this is on purpose
+
+error[E0720]: cannot resolve opaque type
+  --> $DIR/recursive-in-exhaustiveness.rs:27:23
+   |
+LL | fn build2<T>(x: T) -> impl Sized {
+   |                       ^^^^^^^^^^ recursive opaque type
+...
+LL |     (build2(x),)
+   |     ------------ returning here with type `(impl Sized,)`
+
+warning: function cannot return without recursing
+  --> $DIR/recursive-in-exhaustiveness.rs:40:1
+   |
+LL | fn build3<T>(x: T) -> impl Sized {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cannot return without recursing
+LL |
+LL |     let (x,) = (build3((x,)),);
+   |                 ------------ recursive call site
+   |
+   = help: a `loop` may express intention better if this is on purpose
+
+error[E0792]: expected generic type parameter, found `(T,)`
+  --> $DIR/recursive-in-exhaustiveness.rs:49:5
+   |
+LL | fn build3<T>(x: T) -> impl Sized {
+   |           - this generic parameter must be used with a generic type parameter
+...
+LL |     build3(x)
+   |     ^^^^^^^^^
+
+error: aborting due to 2 previous errors; 3 warnings emitted
+
+Some errors have detailed explanations: E0720, E0792.
+For more information about an error, try `rustc --explain E0720`.

tests/ui/impl-trait/recursive-in-exhaustiveness.next.stderr

@@ -0,0 +1,80 @@
+error[E0284]: type annotations needed: cannot satisfy `impl Sized == _`
+  --> $DIR/recursive-in-exhaustiveness.rs:19:17
+   |
+LL |     let (x,) = (build(x),);
+   |                 ^^^^^^^^ cannot satisfy `impl Sized == _`
+
+error[E0271]: type mismatch resolving `build2<(_,)>::{opaque#0} normalizes-to _`
+  --> $DIR/recursive-in-exhaustiveness.rs:31:6
+   |
+LL |     (build2(x),)
+   |      ^^^^^^^^^ types differ
+
+error[E0271]: type mismatch resolving `build2<(_,)>::{opaque#0} normalizes-to _`
+  --> $DIR/recursive-in-exhaustiveness.rs:31:5
+   |
+LL |     (build2(x),)
+   |     ^^^^^^^^^^^^ types differ
+
+error[E0277]: the size for values of type `(impl Sized,)` cannot be known at compilation time
+  --> $DIR/recursive-in-exhaustiveness.rs:31:5
+   |
+LL |     (build2(x),)
+   |     ^^^^^^^^^^^^ doesn't have a size known at compile-time
+   |
+   = help: the trait `Sized` is not implemented for `(impl Sized,)`
+   = note: tuples must have a statically known size to be initialized
+
+error[E0271]: type mismatch resolving `build3<(T,)>::{opaque#0} normalizes-to _`
+  --> $DIR/recursive-in-exhaustiveness.rs:42:17
+   |
+LL |     let (x,) = (build3((x,)),);
+   |                 ^^^^^^^^^^^^ types differ
+
+error[E0277]: the size for values of type `(impl Sized,)` cannot be known at compilation time
+  --> $DIR/recursive-in-exhaustiveness.rs:42:16
+   |
+LL |     let (x,) = (build3((x,)),);
+   |                ^^^^^^^^^^^^^^^ doesn't have a size known at compile-time
+   |
+   = help: the trait `Sized` is not implemented for `(impl Sized,)`
+   = note: tuples must have a statically known size to be initialized
+
+error[E0308]: mismatched types
+  --> $DIR/recursive-in-exhaustiveness.rs:42:16
+   |
+LL | fn build3<T>(x: T) -> impl Sized {
+   |                       ---------- the found opaque type
+LL |
+LL |     let (x,) = (build3((x,)),);
+   |                ^^^^^^^^^^^^^^^ types differ
+   |
+   = note: expected type `_`
+             found tuple `(impl Sized,)`
+
+error[E0271]: type mismatch resolving `build3<(T,)>::{opaque#0} normalizes-to _`
+  --> $DIR/recursive-in-exhaustiveness.rs:42:17
+   |
+LL |     let (x,) = (build3((x,)),);
+   |                 ^^^^^^^^^^^^ types differ
+   |
+   = note: the return type of a function must have a statically known size
+
+error[E0271]: type mismatch resolving `build3<(T,)>::{opaque#0} normalizes-to _`
+  --> $DIR/recursive-in-exhaustiveness.rs:42:16
+   |
+LL |     let (x,) = (build3((x,)),);
+   |                ^^^^^^^^^^^^^^^ types differ
+
+error[E0271]: type mismatch resolving `build3<(T,)>::{opaque#0} normalizes-to _`
+  --> $DIR/recursive-in-exhaustiveness.rs:42:17
+   |
+LL |     let (x,) = (build3((x,)),);
+   |                 ^^^^^^^^^^^^ types differ
+   |
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+error: aborting due to 10 previous errors
+
+Some errors have detailed explanations: E0271, E0277, E0284, E0308.
+For more information about an error, try `rustc --explain E0271`.

tests/ui/impl-trait/recursive-in-exhaustiveness.rs

@@ -0,0 +1,53 @@
+//@ revisions: current next
+//@ ignore-compare-mode-next-solver (explicit revisions)
+//@[next] compile-flags: -Znext-solver
+
+// Test several spicy non-trivial recursive opaque definitions inferred from HIR typeck
+// don't cause stack overflows in exhaustiveness code, which currently reveals opaques
+// manually in a way that is not overflow aware.
+//
+// These should eventually be outright rejected, but today (some) non-trivial recursive
+// opaque definitions are accepted, and changing that requires an FCP, so for now just
+// make sure we don't stack overflow :^)
+
+// Opaque<T> = Opaque<Opaque<T>>
+//
+// We unfortunately accept this today, and due to how opaque type relating is implemented
+// in the NLL type relation, this defines `Opaque<T> = T`.
+fn build<T>(x: T) -> impl Sized {
+    //[current]~^ WARN function cannot return without recursing
+    let (x,) = (build(x),);
+    //[next]~^ ERROR type annotations needed
+    build(x)
+}
+
+// Opaque<T> = (Opaque<T>,)
+//
+// Not allowed today. Detected as recursive.
+fn build2<T>(x: T) -> impl Sized {
+    //[current]~^ ERROR cannot resolve opaque type
+    //[current]~| WARN function cannot return without recursing
+    let (x,) = (build2(x),);
+    (build2(x),)
+    //[next]~^ ERROR type mismatch resolving
+    //[next]~| ERROR type mismatch resolving
+    //[next]~| ERROR the size for values of type
+}
+
+// Opaque<T> = Opaque<(T,)>
+//
+// Not allowed today. Detected as not defining.
+fn build3<T>(x: T) -> impl Sized {
+    //[current]~^ WARN function cannot return without recursing
+    let (x,) = (build3((x,)),);
+    //[next]~^ ERROR type mismatch resolving
+    //[next]~| ERROR type mismatch resolving
+    //[next]~| ERROR type mismatch resolving
+    //[next]~| ERROR type mismatch resolving
+    //[next]~| ERROR the size for values of type
+    //[next]~| ERROR mismatched types
+    build3(x)
+    //[current]~^ ERROR expected generic type parameter, found `(T,)`
+}
+
+fn main() {}