Fix recent crashes in forward refs (#4120)

This adds processing of forward references in two nodes that were 
previously missing: `OverloadedFuncDef` and `NewTypeExpr.info`.

(Note that some tests use `# type: ignore`, the original crash was 
discovered during serialization with errors ignored.)

Fixes #4097.

Author: ilevkivskyi

mypy/nodes.py

@@ -1795,6 +1795,7 @@ class NewTypeExpr(Expression):
     def __init__(self, name: str, old_type: 'Optional[mypy.types.Type]', line: int) -> None:
         self.name = name
         self.old_type = old_type
+        self.line = line
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:
         return visitor.visit_newtype_expr(self)

mypy/semanal_pass3.py

@@ -17,7 +17,7 @@
     Node, Expression, MypyFile, FuncDef, FuncItem, Decorator, RefExpr, Context, TypeInfo, ClassDef,
     Block, TypedDictExpr, NamedTupleExpr, AssignmentStmt, IndexExpr, TypeAliasExpr, NameExpr,
     CallExpr, NewTypeExpr, ForStmt, WithStmt, CastExpr, TypeVarExpr, TypeApplication, Lvalue,
-    TupleExpr, RevealTypeExpr, SymbolTableNode, Var, ARG_POS
+    TupleExpr, RevealTypeExpr, SymbolTableNode, Var, ARG_POS, OverloadedFuncDef
 )
 from mypy.types import (
     Type, Instance, AnyType, TypeOfAny, CallableType, TupleType, TypeVarType, TypedDictType,
@@ -85,6 +85,10 @@ def visit_func_def(self, fdef: FuncDef) -> None:
         super().visit_func_def(fdef)
         self.errors.pop_function()
 
+    def visit_overloaded_func_def(self, fdef: OverloadedFuncDef) -> None:
+        self.analyze(fdef.type, fdef)
+        super().visit_overloaded_func_def(fdef)
+
     def visit_class_def(self, tdef: ClassDef) -> None:
         # NamedTuple base classes are validated in check_namedtuple_classdef; we don't have to
         # check them again here.
@@ -116,12 +120,7 @@ def visit_class_def(self, tdef: ClassDef) -> None:
                 self.analyze(tdef.analyzed.info.typeddict_type, tdef.analyzed, warn=True)
             elif isinstance(tdef.analyzed, NamedTupleExpr):
                 self.analyze(tdef.analyzed.info.tuple_type, tdef.analyzed, warn=True)
-                for name in tdef.analyzed.info.names:
-                    sym = tdef.analyzed.info.names[name]
-                    if isinstance(sym.node, (FuncDef, Decorator)):
-                        self.accept(sym.node)
-                    if isinstance(sym.node, Var):
-                        self.analyze(sym.node.type, sym.node)
+                self.analyze_info(tdef.analyzed.info)
         super().visit_class_def(tdef)
 
     def visit_decorator(self, dec: Decorator) -> None:
@@ -189,6 +188,10 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
             analyzed = s.rvalue.analyzed
             if isinstance(analyzed, NewTypeExpr):
                 self.analyze(analyzed.old_type, analyzed)
+                if analyzed.info:
+                    # Currently NewTypes only have __init__, but to be future proof,
+                    # we analyze all symbols.
+                    self.analyze_info(analyzed.info)
                 if analyzed.info and analyzed.info.mro:
                     analyzed.info.mro = []  # Force recomputation
                     mypy.semanal.calculate_class_mro(analyzed.info.defn, self.fail_blocker)
@@ -203,12 +206,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
                 self.analyze(analyzed.info.typeddict_type, analyzed, warn=True)
             if isinstance(analyzed, NamedTupleExpr):
                 self.analyze(analyzed.info.tuple_type, analyzed, warn=True)
-                for name in analyzed.info.names:
-                    sym = analyzed.info.names[name]
-                    if isinstance(sym.node, (FuncDef, Decorator)):
-                        self.accept(sym.node)
-                    if isinstance(sym.node, Var):
-                        self.analyze(sym.node.type, sym.node)
+                self.analyze_info(analyzed.info)
         # We need to pay additional attention to assignments that define a type alias.
         # The resulting type is also stored in the 'type_override' attribute of
         # the corresponding SymbolTableNode.
@@ -255,12 +253,27 @@ def perform_transform(self, node: Union[Node, SymbolTableNode],
             for n in node.target:
                 if isinstance(n, NameExpr) and isinstance(n.node, Var) and n.node.type:
                     n.node.type = transform(n.node.type)
-        if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)):
+        if isinstance(node, (FuncDef, OverloadedFuncDef, CastExpr, AssignmentStmt,
+                             TypeAliasExpr, Var)):
             assert node.type, "Scheduled patch for non-existent type"
             node.type = transform(node.type)
         if isinstance(node, NewTypeExpr):
             assert node.old_type, "Scheduled patch for non-existent type"
             node.old_type = transform(node.old_type)
+            if node.info:
+                new_bases = []  # type: List[Instance]
+                for b in node.info.bases:
+                    new_b = transform(b)
+                    # TODO: this code can be combined with code in second pass.
+                    if isinstance(new_b, Instance):
+                        new_bases.append(new_b)
+                    elif isinstance(new_b, TupleType):
+                        new_bases.append(new_b.fallback)
+                    else:
+                        self.fail("Argument 2 to NewType(...) must be subclassable"
+                                  " (got {})".format(new_b), node)
+                        new_bases.append(self.builtin_type('object'))
+                node.info.bases = new_bases
         if isinstance(node, TypeVarExpr):
             if node.upper_bound:
                 node.upper_bound = transform(node.upper_bound)
@@ -292,7 +305,7 @@ def perform_transform(self, node: Union[Node, SymbolTableNode],
                     new_bases.append(new_base)
                 else:
                     # Don't fix the NamedTuple bases, they are Instance's intentionally.
-                    # Patch the 'args' just in case, although generic tuple type are
+                    # Patch the 'args' just in case, although generic tuple types are
                     # not supported yet.
                     alt_base = Instance(base.type, [transform(a) for a in base.args])
                     new_bases.append(alt_base)
@@ -339,6 +352,15 @@ def patch() -> None:
                                                                   node, warn=False)))
             self.patches.append(patch)
 
+    def analyze_info(self, info: TypeInfo) -> None:
+        # Similar to above but for nodes with synthetic TypeInfos (NamedTuple and NewType).
+        for name in info.names:
+            sym = info.names[name]
+            if isinstance(sym.node, (FuncDef, Decorator)):
+                self.accept(sym.node)
+            if isinstance(sym.node, Var):
+                self.analyze(sym.node.type, sym.node)
+
     def make_type_analyzer(self, indicator: Dict[str, bool]) -> TypeAnalyserPass3:
         return TypeAnalyserPass3(self.sem.lookup_qualified,
                                  self.sem.lookup_fully_qualified,

mypy/typeanal.py

@@ -14,7 +14,7 @@
     Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType,
     CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, SyntheticTypeVisitor,
     StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args,
-    CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef
+    CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef, Overloaded
 )
 
 from mypy.nodes import (
@@ -766,6 +766,10 @@ def visit_callable_type(self, t: CallableType) -> None:
         for arg_type in t.arg_types:
             arg_type.accept(self)
 
+    def visit_overloaded(self, t: Overloaded) -> None:
+        for item in t.items():
+            item.accept(self)
+
     def visit_tuple_type(self, t: TupleType) -> None:
         for item in t.items:
             item.accept(self)

test-data/unit/check-classes.test

@@ -3623,6 +3623,55 @@ def parse_ast(name_dict: NameDict) -> None:
 [builtins fixtures/isinstancelist.pyi]
 [out]
 
+[case testNoCrashForwardRefToBrokenDoubleNewType]
+from typing import Any, Dict, List, NewType
+
+Foo = NewType('NotFoo', int) # E: String argument 1 'NotFoo' to NewType(...) does not match variable name 'Foo'
+Foos = NewType('Foos', List[Foo]) # type: ignore
+
+def frob(foos: Dict[Any, Foos]) -> None:
+    foo = foos.get(1)
+    dict(foo)
+[builtins fixtures/dict.pyi]
+[out]
+
+[case testNoCrashForwardRefToBrokenDoubleNewTypeClass]
+from typing import Any, Dict, List, NewType
+
+Foo = NewType('NotFoo', int) # type: ignore
+Foos = NewType('Foos', List[Foo]) # type: ignore
+
+x: C
+class C:
+    def frob(self, foos: Dict[Any, Foos]) -> None:
+        foo = foos.get(1)
+        dict(foo)
+
+reveal_type(x.frob) # E: Revealed type is 'def (foos: builtins.dict[Any, __main__.Foos])'
+[builtins fixtures/dict.pyi]
+[out]
+
+[case testNewTypeFromForwardNamedTuple]
+from typing import NewType, NamedTuple, Tuple
+
+NT = NewType('NT', N)
+class N(NamedTuple):
+    x: int
+
+x: NT = N(1) # E: Incompatible types in assignment (expression has type "N", variable has type "NT")
+x = NT(N(1))
+[out]
+
+[case testNewTypeFromForwardTypedDict]
+from typing import NewType, Tuple
+from mypy_extensions import TypedDict
+
+NT = NewType('NT', N) # E: Argument 2 to NewType(...) must be subclassable (got TypedDict('__main__.N', {'x': builtins.int}))
+class N(TypedDict):
+    x: int
+[builtins fixtures/dict.pyi]
+[out]
+
 [case testCorrectAttributeInForwardRefToNamedTuple]
 from typing import NamedTuple
 proc: Process

test-data/unit/check-incremental.test

@@ -3046,6 +3046,71 @@ class Pair(NamedTuple):
 Person(name=Pair(first="John", last="Doe"))
 [out]
 
+[case testNoCrashForwardRefToBrokenDoubleNewTypeIncremental]
+from typing import Any, List, NewType
+
+Foo = NewType('NotFoo', int) # type: ignore
+Foos = NewType('Foos', List[Foo]) # type: ignore
+
+def frob(foos: List[Foos]) -> None:
+    pass
+[builtins fixtures/list.pyi]
+[out]
+
+[case testNoCrashForwardRefOverloadIncremental]
+from typing import overload, List
+
+@overload
+def f(x: int) -> int: ...
+@overload
+def f(x: F) -> F: ...
+def f(x):
+    pass
+
+F = List[int]
+[builtins fixtures/list.pyi]
+[out]
+
+[case testNoCrashForwardRefOverloadIncrementalClass]
+from typing import overload, Tuple, NamedTuple
+
+x: C
+class C:
+    @overload
+    def f(self, x: str) -> N: pass
+    @overload
+    def f(self, x: int) -> int: pass
+    def f(self, x):
+        pass
+
+class N(NamedTuple):
+    x: A
+A = Tuple[int]
+[builtins fixtures/tuple.pyi]
+[out]
+
+[case testNewTypeFromForwardNamedTupleIncremental]
+from typing import NewType, NamedTuple, Tuple
+
+NT = NewType('NT', N)
+class N(NamedTuple):
+    x: int
+
+x: NT = N(1) # type: ignore
+x = NT(N(1))
+[out]
+
+[case testNewTypeFromForwardTypedDictIncremental]
+from typing import NewType, Tuple, Dict
+from mypy_extensions import TypedDict
+
+NT = NewType('NT', N) # type: ignore
+class N(TypedDict):
+    x: A
+A = Dict[str, int]
+[builtins fixtures/dict.pyi]
+[out]
+
 -- Some crazy selef-referential named tuples, types dicts, and aliases
 -- to be sure that everything can be _serialized_ (i.e. ForwardRef's are removed).
 -- For this reason errors are silenced (tests with # type: ignore have equivalents in other files)