Implementing background infrastructure for recursive types: Part 3 (#7885)

This is the last part of plumbing for recursive types (previous #7366 and #7330). Here I implement visitors and related functions.

I convinced myself that we need to only be more careful when a recursive type is checked against another recursive one, so I only special-case these. Logic is similar to how protocols behave, because very roughly type alias can be imagined as a protocol with single property:
```python
A = Union[T, Tuple[A[T], ...]]

class A(Protocol[T]):
    @Property
    def __base__(self) -> Union[T, Tuple[A[T], ...]]: ...
```
but where `TypeAliasType` plays role of `Instance` and `TypeAlias` plays role of `TypeInfo`.

Next two pull requests will contain some non-trivial implementation logic.

Author: ilevkivskyi

mypy/binder.py

@@ -296,16 +296,17 @@ def assign_type(self, expr: Expression,
         # (See discussion in #3526)
         elif (isinstance(type, AnyType)
               and isinstance(declared_type, UnionType)
-              and any(isinstance(item, NoneType) for item in declared_type.items)
+              and any(isinstance(get_proper_type(item), NoneType) for item in declared_type.items)
               and isinstance(get_proper_type(self.most_recent_enclosing_type(expr, NoneType())),
                              NoneType)):
             # Replace any Nones in the union type with Any
-            new_items = [type if isinstance(item, NoneType) else item
+            new_items = [type if isinstance(get_proper_type(item), NoneType) else item
                          for item in declared_type.items]
             self.put(expr, UnionType(new_items))
         elif (isinstance(type, AnyType)
               and not (isinstance(declared_type, UnionType)
-                       and any(isinstance(item, AnyType) for item in declared_type.items))):
+                       and any(isinstance(get_proper_type(item), AnyType)
+                               for item in declared_type.items))):
             # Assigning an Any value doesn't affect the type to avoid false negatives, unless
             # there is an Any item in a declared union type.
             self.put(expr, declared_type)

mypy/checker.py

@@ -36,8 +36,7 @@
     UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef,
     is_named_instance, union_items, TypeQuery, LiteralType,
     is_optional, remove_optional, TypeTranslator, StarType, get_proper_type, ProperType,
-    get_proper_types, is_literal_type
-)
+    get_proper_types, is_literal_type, TypeAliasType)
 from mypy.sametypes import is_same_type
 from mypy.messages import (
     MessageBuilder, make_inferred_type_note, append_invariance_notes,
@@ -2480,7 +2479,7 @@ def check_multi_assignment(self, lvalues: List[Lvalue],
             # If this is an Optional type in non-strict Optional code, unwrap it.
             relevant_items = rvalue_type.relevant_items()
             if len(relevant_items) == 1:
-                rvalue_type = relevant_items[0]
+                rvalue_type = get_proper_type(relevant_items[0])
 
         if isinstance(rvalue_type, AnyType):
             for lv in lvalues:
@@ -2587,7 +2586,7 @@ def check_multi_assignment_from_tuple(self, lvalues: List[Lvalue], rvalue: Expre
                     # If this is an Optional type in non-strict Optional code, unwrap it.
                     relevant_items = reinferred_rvalue_type.relevant_items()
                     if len(relevant_items) == 1:
-                        reinferred_rvalue_type = relevant_items[0]
+                        reinferred_rvalue_type = get_proper_type(relevant_items[0])
                 if isinstance(reinferred_rvalue_type, UnionType):
                     self.check_multi_assignment_from_union(lvalues, rvalue,
                                                            reinferred_rvalue_type, context,
@@ -3732,7 +3731,7 @@ def find_isinstance_check(self, node: Expression
                     type = get_isinstance_type(node.args[1], type_map)
                     if isinstance(vartype, UnionType):
                         union_list = []
-                        for t in vartype.items:
+                        for t in get_proper_types(vartype.items):
                             if isinstance(t, TypeType):
                                 union_list.append(t.item)
                             else:
@@ -4558,6 +4557,7 @@ def overload_can_never_match(signature: CallableType, other: CallableType) -> bo
     # TODO: find a cleaner solution instead of this ad-hoc erasure.
     exp_signature = expand_type(signature, {tvar.id: erase_def_to_union_or_bound(tvar)
                                 for tvar in signature.variables})
+    assert isinstance(exp_signature, ProperType)
     assert isinstance(exp_signature, CallableType)
     return is_callable_compatible(exp_signature, other,
                                   is_compat=is_more_precise,
@@ -4641,6 +4641,11 @@ def visit_uninhabited_type(self, t: UninhabitedType) -> Type:
             return AnyType(TypeOfAny.from_error)
         return t
 
+    def visit_type_alias_type(self, t: TypeAliasType) -> Type:
+        # Target of the alias cannot by an ambigous <nothing>, so we just
+        # replace the arguments.
+        return t.copy_modified(args=[a.accept(self) for a in t.args])
+
 
 def is_node_static(node: Optional[Node]) -> Optional[bool]:
     """Find out if a node describes a static function method."""

mypy/checkexpr.py

@@ -900,7 +900,7 @@ def check_callable_call(self,
             callee = callee.copy_modified(ret_type=new_ret_type)
         return callee.ret_type, callee
 
-    def analyze_type_type_callee(self, item: ProperType, context: Context) -> ProperType:
+    def analyze_type_type_callee(self, item: ProperType, context: Context) -> Type:
         """Analyze the callee X in X(...) where X is Type[item].
 
         Return a Y that we can pass to check_call(Y, ...).
@@ -913,14 +913,15 @@ def analyze_type_type_callee(self, item: ProperType, context: Context) -> Proper
                 res = res.copy_modified(from_type_type=True)
             return expand_type_by_instance(res, item)
         if isinstance(item, UnionType):
-            return UnionType([self.analyze_type_type_callee(tp, context)
+            return UnionType([self.analyze_type_type_callee(get_proper_type(tp), context)
                               for tp in item.relevant_items()], item.line)
         if isinstance(item, TypeVarType):
             # Pretend we're calling the typevar's upper bound,
             # i.e. its constructor (a poor approximation for reality,
             # but better than AnyType...), but replace the return type
             # with typevar.
             callee = self.analyze_type_type_callee(get_proper_type(item.upper_bound), context)
+            callee = get_proper_type(callee)
             if isinstance(callee, CallableType):
                 callee = callee.copy_modified(ret_type=item)
             elif isinstance(callee, Overloaded):
@@ -2144,8 +2145,7 @@ def dangerous_comparison(self, left: Type, right: Type,
         if not self.chk.options.strict_equality:
             return False
 
-        left = get_proper_type(left)
-        right = get_proper_type(right)
+        left, right = get_proper_types((left, right))
 
         if self.chk.binder.is_unreachable_warning_suppressed():
             # We are inside a function that contains type variables with value restrictions in
@@ -2165,6 +2165,7 @@ def dangerous_comparison(self, left: Type, right: Type,
         if isinstance(left, UnionType) and isinstance(right, UnionType):
             left = remove_optional(left)
             right = remove_optional(right)
+            left, right = get_proper_types((left, right))
         py2 = self.chk.options.python_version < (3, 0)
         if (original_container and has_bytes_component(original_container, py2) and
                 has_bytes_component(left, py2)):
@@ -2794,7 +2795,7 @@ def try_getting_int_literals(self, index: Expression) -> Optional[List[int]]:
             return [typ.value]
         if isinstance(typ, UnionType):
             out = []
-            for item in typ.items:
+            for item in get_proper_types(typ.items):
                 if isinstance(item, LiteralType) and isinstance(item.value, int):
                     out.append(item.value)
                 else:
@@ -2969,7 +2970,7 @@ class LongName(Generic[T]): ...
         # For example:
         #     A = List[Tuple[T, T]]
         #     x = A() <- same as List[Tuple[Any, Any]], see PEP 484.
-        item = set_any_tvars(target, alias_tvars, ctx.line, ctx.column)
+        item = get_proper_type(set_any_tvars(target, alias_tvars, ctx.line, ctx.column))
         if isinstance(item, Instance):
             # Normally we get a callable type (or overloaded) with .is_type_obj() true
             # representing the class's constructor
@@ -3052,7 +3053,7 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type:
         type_context = get_proper_type(self.type_context[-1])
         type_context_items = None
         if isinstance(type_context, UnionType):
-            tuples_in_context = [t for t in type_context.items
+            tuples_in_context = [t for t in get_proper_types(type_context.items)
                                  if (isinstance(t, TupleType) and len(t.items) == len(e.items)) or
                                  is_named_instance(t, 'builtins.tuple')]
             if len(tuples_in_context) == 1:
@@ -3240,7 +3241,8 @@ def infer_lambda_type_using_context(self, e: LambdaExpr) -> Tuple[Optional[Calla
         ctx = get_proper_type(self.type_context[-1])
 
         if isinstance(ctx, UnionType):
-            callables = [t for t in ctx.relevant_items() if isinstance(t, CallableType)]
+            callables = [t for t in get_proper_types(ctx.relevant_items())
+                         if isinstance(t, CallableType)]
             if len(callables) == 1:
                 ctx = callables[0]
 

mypy/checkmember.py

@@ -6,7 +6,7 @@
 from mypy.types import (
     Type, Instance, AnyType, TupleType, TypedDictType, CallableType, FunctionLike, TypeVarDef,
     Overloaded, TypeVarType, UnionType, PartialType, TypeOfAny, LiteralType,
-    DeletedType, NoneType, TypeType, get_type_vars, get_proper_type, ProperType
+    DeletedType, NoneType, TypeType, has_type_vars, get_proper_type, ProperType
 )
 from mypy.nodes import (
     TypeInfo, FuncBase, Var, FuncDef, SymbolNode, Context, MypyFile, TypeVarExpr,
@@ -377,7 +377,7 @@ def analyze_member_var_access(name: str,
                     function = function_type(method, mx.builtin_type('builtins.function'))
                     bound_method = bind_self(function, mx.self_type)
                     typ = map_instance_to_supertype(itype, method.info)
-                    getattr_type = expand_type_by_instance(bound_method, typ)
+                    getattr_type = get_proper_type(expand_type_by_instance(bound_method, typ))
                     if isinstance(getattr_type, CallableType):
                         result = getattr_type.ret_type
 
@@ -394,7 +394,7 @@ def analyze_member_var_access(name: str,
                 setattr_func = function_type(setattr_meth, mx.builtin_type('builtins.function'))
                 bound_type = bind_self(setattr_func, mx.self_type)
                 typ = map_instance_to_supertype(itype, setattr_meth.info)
-                setattr_type = expand_type_by_instance(bound_type, typ)
+                setattr_type = get_proper_type(expand_type_by_instance(bound_type, typ))
                 if isinstance(setattr_type, CallableType) and len(setattr_type.arg_types) > 0:
                     return setattr_type.arg_types[-1]
 
@@ -497,10 +497,11 @@ def instance_alias_type(alias: TypeAlias,
 
     As usual, we first erase any unbound type variables to Any.
     """
-    target = get_proper_type(alias.target)
-    assert isinstance(target, Instance), "Must be called only with aliases to classes"
+    target = get_proper_type(alias.target)  # type: Type
+    assert isinstance(get_proper_type(target),
+                      Instance), "Must be called only with aliases to classes"
     target = set_any_tvars(target, alias.alias_tvars, alias.line, alias.column)
-    assert isinstance(target, Instance)
+    assert isinstance(target, Instance)  # type: ignore[misc]
     tp = type_object_type(target.type, builtin_type)
     return expand_type_by_instance(tp, target)
 
@@ -525,7 +526,7 @@ def analyze_var(name: str,
     if typ:
         if isinstance(typ, PartialType):
             return mx.chk.handle_partial_var_type(typ, mx.is_lvalue, var, mx.context)
-        t = expand_type_by_instance(typ, itype)
+        t = get_proper_type(expand_type_by_instance(typ, itype))
         if mx.is_lvalue and var.is_property and not var.is_settable_property:
             # TODO allow setting attributes in subclass (although it is probably an error)
             mx.msg.read_only_property(name, itype.type, mx.context)
@@ -577,7 +578,9 @@ def analyze_var(name: str,
     return result
 
 
-def freeze_type_vars(member_type: ProperType) -> None:
+def freeze_type_vars(member_type: Type) -> None:
+    if not isinstance(member_type, ProperType):
+        return
     if isinstance(member_type, CallableType):
         for v in member_type.variables:
             v.id.meta_level = 0
@@ -713,7 +716,7 @@ def analyze_class_attribute_access(itype: Instance,
             #         x: T
             #     C.x  # Error, ambiguous access
             #     C[int].x  # Also an error, since C[int] is same as C at runtime
-            if isinstance(t, TypeVarType) or get_type_vars(t):
+            if isinstance(t, TypeVarType) or has_type_vars(t):
                 # Exception: access on Type[...], including first argument of class methods is OK.
                 if not isinstance(get_proper_type(mx.original_type), TypeType):
                     if node.node.is_classvar:
@@ -799,7 +802,7 @@ class B(A[str]): pass
     info = itype.type  # type: TypeInfo
     if is_classmethod:
         assert isuper is not None
-        t = expand_type_by_instance(t, isuper)
+        t = get_proper_type(expand_type_by_instance(t, isuper))
     # We add class type variables if the class method is accessed on class object
     # without applied type arguments, this matches the behavior of __init__().
     # For example (continuing the example in docstring):

mypy/checkstrformat.py

@@ -19,7 +19,7 @@
 
 from mypy.types import (
     Type, AnyType, TupleType, Instance, UnionType, TypeOfAny, get_proper_type, TypeVarType,
-    CallableType, LiteralType
+    CallableType, LiteralType, get_proper_types
 )
 from mypy.nodes import (
     StrExpr, BytesExpr, UnicodeExpr, TupleExpr, DictExpr, Context, Expression, StarExpr, CallExpr,
@@ -359,7 +359,8 @@ def check_specs_in_format_call(self, call: CallExpr,
                 continue
 
             a_type = get_proper_type(actual_type)
-            actual_items = a_type.items if isinstance(a_type, UnionType) else [a_type]
+            actual_items = (get_proper_types(a_type.items) if isinstance(a_type, UnionType)
+                            else [a_type])
             for a_type in actual_items:
                 if custom_special_method(a_type, '__format__'):
                     continue

mypy/constraints.py

@@ -7,7 +7,7 @@
     CallableType, Type, TypeVisitor, UnboundType, AnyType, NoneType, TypeVarType, Instance,
     TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, DeletedType,
     UninhabitedType, TypeType, TypeVarId, TypeQuery, is_named_instance, TypeOfAny, LiteralType,
-    ProperType, get_proper_type
+    ProperType, get_proper_type, TypeAliasType
 )
 from mypy.maptype import map_instance_to_supertype
 import mypy.subtypes
@@ -16,6 +16,7 @@
 from mypy.erasetype import erase_typevars
 from mypy.nodes import COVARIANT, CONTRAVARIANT
 from mypy.argmap import ArgTypeExpander
+from mypy.typestate import TypeState
 
 SUBTYPE_OF = 0  # type: Final[int]
 SUPERTYPE_OF = 1  # type: Final[int]
@@ -89,6 +90,21 @@ def infer_constraints(template: Type, actual: Type,
 
     The constraints are represented as Constraint objects.
     """
+    if any(get_proper_type(template) == get_proper_type(t) for t in TypeState._inferring):
+        return []
+    if (isinstance(template, TypeAliasType) and isinstance(actual, TypeAliasType) and
+            template.is_recursive and actual.is_recursive):
+        # This case requires special care because it may cause infinite recursion.
+        TypeState._inferring.append(template)
+        res = _infer_constraints(template, actual, direction)
+        TypeState._inferring.pop()
+        return res
+    return _infer_constraints(template, actual, direction)
+
+
+def _infer_constraints(template: Type, actual: Type,
+                       direction: int) -> List[Constraint]:
+
     template = get_proper_type(template)
     actual = get_proper_type(actual)
 
@@ -487,6 +503,9 @@ def visit_union_type(self, template: UnionType) -> List[Constraint]:
         assert False, ("Unexpected UnionType in ConstraintBuilderVisitor"
                        " (should have been handled in infer_constraints)")
 
+    def visit_type_alias_type(self, template: TypeAliasType) -> List[Constraint]:
+        assert False, "This should be never called, got {}".format(template)
+
     def infer_against_any(self, types: Iterable[Type], any_type: AnyType) -> List[Constraint]:
         res = []  # type: List[Constraint]
         for t in types:

mypy/erasetype.py

@@ -4,7 +4,7 @@
     Type, TypeVisitor, UnboundType, AnyType, NoneType, TypeVarId, Instance, TypeVarType,
     CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType,
     DeletedType, TypeTranslator, UninhabitedType, TypeType, TypeOfAny, LiteralType, ProperType,
-    get_proper_type
+    get_proper_type, TypeAliasType
 )
 from mypy.nodes import ARG_STAR, ARG_STAR2
 
@@ -93,6 +93,9 @@ def visit_union_type(self, t: UnionType) -> ProperType:
     def visit_type_type(self, t: TypeType) -> ProperType:
         return TypeType.make_normalized(t.item.accept(self), line=t.line)
 
+    def visit_type_alias_type(self, t: TypeAliasType) -> ProperType:
+        raise RuntimeError("Type aliases should be expanded before accepting this visitor")
+
 
 def erase_typevars(t: Type, ids_to_erase: Optional[Container[TypeVarId]] = None) -> Type:
     """Replace all type variables in a type with any,
@@ -122,6 +125,11 @@ def visit_type_var(self, t: TypeVarType) -> Type:
             return self.replacement
         return t
 
+    def visit_type_alias_type(self, t: TypeAliasType) -> Type:
+        # Type alias target can't contain bound type variables, so
+        # it is safe to just erase the arguments.
+        return t.copy_modified(args=[a.accept(self) for a in t.args])
+
 
 def remove_instance_last_known_values(t: Type) -> Type:
     return t.accept(LastKnownValueEraser())
@@ -135,3 +143,8 @@ def visit_instance(self, t: Instance) -> Type:
         if t.last_known_value:
             return t.copy_modified(last_known_value=None)
         return t
+
+    def visit_type_alias_type(self, t: TypeAliasType) -> Type:
+        # Type aliases can't contain literal values, because they are
+        # always constructed as explicit types.
+        return t