[PEP 747] Recognize TypeForm[T] type and values (python#9773)

User must opt-in to use TypeForm with --enable-incomplete-feature=TypeForm

In particular:
* Recognize TypeForm[T] as a kind of type that can be used in a type expression
* Recognize a type expression literal as a TypeForm value in:
    - assignments
    - function calls
    - return statements
* Define the following relationships between TypeForm values:
    - is_subtype
    - join_types
    - meet_types
* Recognize the TypeForm(...) expression
* Alter isinstance(typx, type) to narrow TypeForm[T] to Type[T]

Author: davidfstr

Diff for: mypy/checker.py

@@ -7459,7 +7459,10 @@ def add_any_attribute_to_type(self, typ: Type, name: str) -> Type:
             fallback = typ.fallback.copy_with_extra_attr(name, any_type)
             return typ.copy_modified(fallback=fallback)
         if isinstance(typ, TypeType) and isinstance(typ.item, Instance):
-            return TypeType.make_normalized(self.add_any_attribute_to_type(typ.item, name))
+            return TypeType.make_normalized(
+                self.add_any_attribute_to_type(typ.item, name),
+                is_type_form=typ.is_type_form,
+            )
         if isinstance(typ, TypeVarType):
             return typ.copy_modified(
                 upper_bound=self.add_any_attribute_to_type(typ.upper_bound, name),

Diff for: mypy/checkexpr.py

@@ -90,6 +90,7 @@
     TypeAliasExpr,
     TypeApplication,
     TypedDictExpr,
+    TypeFormExpr,
     TypeInfo,
     TypeVarExpr,
     TypeVarTupleExpr,
@@ -4601,6 +4602,10 @@ def visit_cast_expr(self, expr: CastExpr) -> Type:
         )
         return target_type
 
+    def visit_type_form_expr(self, expr: TypeFormExpr) -> Type:
+        typ = expr.type
+        return TypeType.make_normalized(typ, line=typ.line, column=typ.column, is_type_form=True)
+
     def visit_assert_type_expr(self, expr: AssertTypeExpr) -> Type:
         source_type = self.accept(
             expr.expr,
@@ -5831,6 +5836,7 @@ def accept(
         old_is_callee = self.is_callee
         self.is_callee = is_callee
         try:
+            p_type_context = get_proper_type(type_context)
             if allow_none_return and isinstance(node, CallExpr):
                 typ = self.visit_call_expr(node, allow_none_return=True)
             elif allow_none_return and isinstance(node, YieldFromExpr):
@@ -5839,6 +5845,17 @@ def accept(
                 typ = self.visit_conditional_expr(node, allow_none_return=True)
             elif allow_none_return and isinstance(node, AwaitExpr):
                 typ = self.visit_await_expr(node, allow_none_return=True)
+            elif (
+                isinstance(p_type_context, TypeType) and
+                p_type_context.is_type_form and
+                node.as_type is not None
+            ):
+                typ = TypeType.make_normalized(
+                    node.as_type,
+                    line=node.as_type.line,
+                    column=node.as_type.column,
+                    is_type_form=True,
+                )
             else:
                 typ = node.accept(self)
         except Exception as err:

Diff for: mypy/copytype.py

@@ -120,7 +120,7 @@ def visit_overloaded(self, t: Overloaded) -> ProperType:
 
     def visit_type_type(self, t: TypeType) -> ProperType:
         # Use cast since the type annotations in TypeType are imprecise.
-        return self.copy_common(t, TypeType(cast(Any, t.item)))
+        return self.copy_common(t, TypeType(cast(Any, t.item), is_type_form=t.is_type_form))
 
     def visit_type_alias_type(self, t: TypeAliasType) -> ProperType:
         assert False, "only ProperTypes supported"

Diff for: mypy/erasetype.py

@@ -133,7 +133,7 @@ def visit_union_type(self, t: UnionType) -> ProperType:
         return make_simplified_union(erased_items)
 
     def visit_type_type(self, t: TypeType) -> ProperType:
-        return TypeType.make_normalized(t.item.accept(self), line=t.line)
+        return TypeType.make_normalized(t.item.accept(self), line=t.line, is_type_form=t.is_type_form)
 
     def visit_type_alias_type(self, t: TypeAliasType) -> ProperType:
         raise RuntimeError("Type aliases should be expanded before accepting this visitor")

Diff for: mypy/evalexpr.py

@@ -75,6 +75,9 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> object:
     def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> object:
         return o.expr.accept(self)
 
+    def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr) -> object:
+        return UNKNOWN
+
     def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> object:
         return o.expr.accept(self)
 

Diff for: mypy/expandtype.py

@@ -500,7 +500,7 @@ def visit_type_type(self, t: TypeType) -> Type:
         # union of instances or Any).  Sadly we can't report errors
         # here yet.
         item = t.item.accept(self)
-        return TypeType.make_normalized(item)
+        return TypeType.make_normalized(item, is_type_form=t.is_type_form)
 
     def visit_type_alias_type(self, t: TypeAliasType) -> Type:
         # Target of the type alias cannot contain type variables (not bound by the type

Diff for: mypy/join.py

@@ -657,7 +657,11 @@ def visit_partial_type(self, t: PartialType) -> ProperType:
 
     def visit_type_type(self, t: TypeType) -> ProperType:
         if isinstance(self.s, TypeType):
-            return TypeType.make_normalized(join_types(t.item, self.s.item), line=t.line)
+            return TypeType.make_normalized(
+                join_types(t.item, self.s.item),
+                line=t.line,
+                is_type_form=self.s.is_type_form or t.is_type_form,
+            )
         elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type":
             return self.s
         else:

Diff for: mypy/literals.py

@@ -47,6 +47,7 @@
     TypeAliasExpr,
     TypeApplication,
     TypedDictExpr,
+    TypeFormExpr,
     TypeVarExpr,
     TypeVarTupleExpr,
     UnaryExpr,
@@ -230,6 +231,9 @@ def visit_slice_expr(self, e: SliceExpr) -> None:
     def visit_cast_expr(self, e: CastExpr) -> None:
         return None
 
+    def visit_type_form_expr(self, e: TypeFormExpr) -> None:
+        return None
+
     def visit_assert_type_expr(self, e: AssertTypeExpr) -> None:
         return None
 

Diff for: mypy/meet.py

@@ -157,12 +157,27 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type:
     elif isinstance(narrowed, TypeVarType) and is_subtype(narrowed.upper_bound, declared):
         return narrowed
     elif isinstance(declared, TypeType) and isinstance(narrowed, TypeType):
-        return TypeType.make_normalized(narrow_declared_type(declared.item, narrowed.item))
+        return TypeType.make_normalized(
+            narrow_declared_type(declared.item, narrowed.item),
+            is_type_form=declared.is_type_form and narrowed.is_type_form,
+        )
     elif (
         isinstance(declared, TypeType)
         and isinstance(narrowed, Instance)
         and narrowed.type.is_metaclass()
     ):
+        if declared.is_type_form:
+            # The declared TypeForm[T] after narrowing must be a kind of
+            # type object at least as narrow as Type[T]
+            return narrow_declared_type(
+                TypeType.make_normalized(
+                    declared.item,
+                    line=declared.line,
+                    column=declared.column,
+                    is_type_form=False,
+                ),
+                original_narrowed,
+            )
         # We'd need intersection types, so give up.
         return original_declared
     elif isinstance(declared, Instance):
@@ -1039,7 +1054,11 @@ def visit_type_type(self, t: TypeType) -> ProperType:
         if isinstance(self.s, TypeType):
             typ = self.meet(t.item, self.s.item)
             if not isinstance(typ, NoneType):
-                typ = TypeType.make_normalized(typ, line=t.line)
+                typ = TypeType.make_normalized(
+                    typ,
+                    line=t.line,
+                    is_type_form=self.s.is_type_form and t.is_type_form,
+                )
             return typ
         elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type":
             return t

Diff for: mypy/messages.py

@@ -2675,7 +2675,10 @@ def format_literal_value(typ: LiteralType) -> str:
     elif isinstance(typ, UninhabitedType):
         return "Never"
     elif isinstance(typ, TypeType):
-        type_name = "type" if options.use_lowercase_names() else "Type"
+        if typ.is_type_form:
+            type_name = "TypeForm"
+        else:
+            type_name = "type" if options.use_lowercase_names() else "Type"
         return f"{type_name}[{format(typ.item)}]"
     elif isinstance(typ, FunctionLike):
         func = typ

Diff for: mypy/mixedtraverser.py

@@ -13,6 +13,7 @@
     TypeAliasExpr,
     TypeApplication,
     TypedDictExpr,
+    TypeFormExpr,
     TypeVarExpr,
     Var,
     WithStmt,
@@ -96,6 +97,10 @@ def visit_cast_expr(self, o: CastExpr) -> None:
         super().visit_cast_expr(o)
         o.type.accept(self)
 
+    def visit_type_form_expr(self, o: TypeFormExpr) -> None:
+        super().visit_type_form_expr(o)
+        o.type.accept(self)
+
     def visit_assert_type_expr(self, o: AssertTypeExpr) -> None:
         super().visit_assert_type_expr(o)
         o.type.accept(self)

Diff for: mypy/nodes.py

@@ -212,7 +212,19 @@ def accept(self, visitor: StatementVisitor[T]) -> T:
 class Expression(Node):
     """An expression node."""
 
-    __slots__ = ()
+    # NOTE: Cannot use __slots__ because some subclasses also inherit from
+    #       a different superclass with its own __slots__. A subclass in
+    #       Python is not allowed to have multiple superclasses that define
+    #       __slots__.
+    #__slots__ = ('as_type',)
+
+    # If this value expression can also be parsed as a valid type expression,
+    # represents the type denoted by the type expression.
+    as_type: mypy.types.Type | None
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.as_type = None
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:
         raise RuntimeError("Not implemented", type(self))
@@ -2189,6 +2201,23 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T:
         return visitor.visit_cast_expr(self)
 
 
+class TypeFormExpr(Expression):
+    """TypeForm(type) expression."""
+
+    __slots__ = ("type",)
+
+    __match_args__ = ("type",)
+
+    type: mypy.types.Type
+
+    def __init__(self, typ: mypy.types.Type) -> None:
+        super().__init__()
+        self.type = typ
+
+    def accept(self, visitor: ExpressionVisitor[T]) -> T:
+        return visitor.visit_type_form_expr(self)
+
+
 class AssertTypeExpr(Expression):
     """Represents a typing.assert_type(expr, type) call."""
 

Diff for: mypy/options.py

@@ -75,7 +75,8 @@ class BuildType:
 PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes"
 NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax"
 INLINE_TYPEDDICT: Final = "InlineTypedDict"
-INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT))
+TYPE_FORM: Final = "TypeForm"
+INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT, TYPE_FORM))
 COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, NEW_GENERIC_SYNTAX))
 
 

Diff for: mypy/semanal.py

@@ -167,6 +167,7 @@
     TypeAliasStmt,
     TypeApplication,
     TypedDictExpr,
+    TypeFormExpr,
     TypeInfo,
     TypeParam,
     TypeVarExpr,
@@ -186,7 +187,7 @@
     type_aliases_source_versions,
     typing_extensions_aliases,
 )
-from mypy.options import Options
+from mypy.options import Options, TYPE_FORM
 from mypy.patterns import (
     AsPattern,
     ClassPattern,
@@ -3151,6 +3152,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
         self.store_final_status(s)
         self.check_classvar(s)
         self.process_type_annotation(s)
+        self.analyze_rvalue_as_type_form(s)
         self.apply_dynamic_class_hook(s)
         if not s.type:
             self.process_module_assignment(s.lvalues, s.rvalue, s)
@@ -3479,6 +3481,10 @@ def analyze_lvalues(self, s: AssignmentStmt) -> None:
                 has_explicit_value=has_explicit_value,
             )
 
+    def analyze_rvalue_as_type_form(self, s: AssignmentStmt) -> None:
+        if TYPE_FORM in self.options.enable_incomplete_feature:
+            s.rvalue.as_type = self.try_parse_as_type_expression(s.rvalue)
+
     def apply_dynamic_class_hook(self, s: AssignmentStmt) -> None:
         if not isinstance(s.rvalue, CallExpr):
             return
@@ -5161,6 +5167,8 @@ def visit_return_stmt(self, s: ReturnStmt) -> None:
             self.fail('"return" outside function', s)
         if s.expr:
             s.expr.accept(self)
+            if TYPE_FORM in self.options.enable_incomplete_feature:
+                s.expr.as_type = self.try_parse_as_type_expression(s.expr)
 
     def visit_raise_stmt(self, s: RaiseStmt) -> None:
         self.statement = s
@@ -5674,10 +5682,33 @@ def visit_call_expr(self, expr: CallExpr) -> None:
             with self.allow_unbound_tvars_set():
                 for a in expr.args:
                     a.accept(self)
+        elif refers_to_fullname(
+            expr.callee, ("typing.TypeForm", "typing_extensions.TypeForm")
+        ):
+            # Special form TypeForm(...).
+            if not self.check_fixed_args(expr, 1, "TypeForm"):
+                return
+            # Translate first argument to an unanalyzed type.
+            try:
+                typ = self.expr_to_unanalyzed_type(expr.args[0])
+            except TypeTranslationError:
+                self.fail("TypeForm argument is not a type", expr)
+                # Suppress future error: "<typing special form>" not callable
+                expr.analyzed = CastExpr(expr.args[0], AnyType(TypeOfAny.from_error))
+                return
+            # Piggyback TypeFormExpr object to the CallExpr object; it takes
+            # precedence over the CallExpr semantics.
+            expr.analyzed = TypeFormExpr(typ)
+            expr.analyzed.line = expr.line
+            expr.analyzed.column = expr.column
+            expr.analyzed.accept(self)
         else:
             # Normal call expression.
+            calculate_type_forms = TYPE_FORM in self.options.enable_incomplete_feature
             for a in expr.args:
                 a.accept(self)
+                if calculate_type_forms:
+                    a.as_type = self.try_parse_as_type_expression(a)
 
             if (
                 isinstance(expr.callee, MemberExpr)
@@ -5946,6 +5977,11 @@ def visit_cast_expr(self, expr: CastExpr) -> None:
         if analyzed is not None:
             expr.type = analyzed
 
+    def visit_type_form_expr(self, expr: TypeFormExpr) -> None:
+        analyzed = self.anal_type(expr.type)
+        if analyzed is not None:
+            expr.type = analyzed
+
     def visit_assert_type_expr(self, expr: AssertTypeExpr) -> None:
         expr.expr.accept(self)
         analyzed = self.anal_type(expr.type)
@@ -7381,6 +7417,35 @@ def parse_dataclass_transform_field_specifiers(self, arg: Expression) -> tuple[s
             names.append(specifier.fullname)
         return tuple(names)
 
+    def try_parse_as_type_expression(self, maybe_type_expr: Expression) -> Type|None:
+        """Try to parse maybe_type_expr as a type expression.
+        If parsing fails return None and emit no errors."""
+        # Save SemanticAnalyzer state
+        original_errors = self.errors  # altered by fail()
+        original_num_incomplete_refs = self.num_incomplete_refs  # altered by record_incomplete_ref()
+        original_progress = self.progress  # altered by defer()
+        original_deferred = self.deferred  # altered by defer()
+        original_deferral_debug_context_len = len(self.deferral_debug_context)  # altered by defer()
+
+        self.errors = Errors(Options())
+        try:
+            t = self.expr_to_analyzed_type(maybe_type_expr)
+            if self.errors.is_errors():
+                raise TypeTranslationError
+            if isinstance(t, (UnboundType, PlaceholderType)):  # type: ignore[misc]
+                raise TypeTranslationError
+        except TypeTranslationError:
+            # Not a type expression. It must be a value expression.
+            t = None
+        finally:
+            # Restore SemanticAnalyzer state
+            self.errors = original_errors
+            self.num_incomplete_refs = original_num_incomplete_refs
+            self.progress = original_progress
+            self.deferred = original_deferred
+            del self.deferral_debug_context[original_deferral_debug_context_len:]
+        return t
+
 
 def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike:
     if isinstance(sig, CallableType):

Diff for: mypy/server/astdiff.py

@@ -496,7 +496,7 @@ def visit_partial_type(self, typ: PartialType) -> SnapshotItem:
         raise RuntimeError
 
     def visit_type_type(self, typ: TypeType) -> SnapshotItem:
-        return ("TypeType", snapshot_type(typ.item))
+        return ("TypeType", snapshot_type(typ.item), typ.is_type_form)
 
     def visit_type_alias_type(self, typ: TypeAliasType) -> SnapshotItem:
         assert typ.alias is not None