From 0f8c36cfb44b554f4fc2d01b7e293bd828a53941 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Thu, 2 Feb 2023 14:29:32 +0900 Subject: [PATCH 01/14] Don't duplicate ParamSpec prefixes and check Concatenate subtyping better I ran into the duplication issue personally, but this also fixes https://github.com/python/mypy/issues/12734 --- mypy/constraints.py | 2 +- mypy/expandtype.py | 10 +++- mypy/messages.py | 10 ++-- mypy/subtypes.py | 49 ++++++++++++++++++- mypy/types.py | 28 +++++++---- .../unit/check-parameter-specification.test | 15 ++++++ 6 files changed, 96 insertions(+), 18 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index a8f04094ca63..c8c3c7933b6e 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -949,7 +949,7 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: ) # TODO: see above "FIX" comments for param_spec is None case - # TODO: this assume positional arguments + # TODO: this assumes positional arguments for t, a in zip(prefix.arg_types, cactual_prefix.arg_types): res.extend(infer_constraints(t, a, neg_op(self.direction))) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 7933283b24d6..5d92c474f874 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -234,7 +234,15 @@ def visit_type_var(self, t: TypeVarType) -> Type: return repl def visit_param_spec(self, t: ParamSpecType) -> Type: - repl = get_proper_type(self.variables.get(t.id, t)) + repl_ = self.variables.get(t.id) + if not repl_: + # in this case, we are trying to expand it into... nothing? + # if we continue, we will get e.g.: + # repl = t = [int, str, **P] + # at which point, expand_param_spec will duplicate the arguments. + return t + + repl = get_proper_type(repl_) if isinstance(repl, Instance): # TODO: what does prefix mean in this case? # TODO: why does this case even happen? Instances aren't plural. diff --git a/mypy/messages.py b/mypy/messages.py index a5fd09493456..41adc63530fb 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2399,13 +2399,15 @@ def format_literal_value(typ: LiteralType) -> str: return_type = format(func.ret_type) if func.is_ellipsis_args: return f"Callable[..., {return_type}]" + param_spec = func.param_spec() if param_spec is not None: return f"Callable[{format(param_spec)}, {return_type}]" - args = format_callable_args( - func.arg_types, func.arg_kinds, func.arg_names, format, verbosity - ) - return f"Callable[[{args}], {return_type}]" + else: + args = format_callable_args( + func.arg_types, func.arg_kinds, func.arg_names, format, verbosity + ) + return f"Callable[[{args}], {return_type}]" else: # Use a simple representation for function types; proper # function types may result in long and difficult-to-read diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 9b555480e59b..fc919f498cab 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -388,8 +388,7 @@ def _is_subtype(self, left: Type, right: Type) -> bool: return is_proper_subtype(left, right, subtype_context=self.subtype_context) return is_subtype(left, right, subtype_context=self.subtype_context) - # visit_x(left) means: is left (which is an instance of X) a subtype of - # right? + # visit_x(left) means: is left (which is an instance of X) a subtype of right? def visit_unbound_type(self, left: UnboundType) -> bool: # This can be called if there is a bad type annotation. The result probably @@ -643,6 +642,9 @@ def visit_param_spec(self, left: ParamSpecType) -> bool: isinstance(right, ParamSpecType) and right.id == left.id and right.flavor == left.flavor + and is_subtype( + left.as_parameters(), right.as_parameters(), subtype_context=self.subtype_context + ) ): return True if isinstance(right, Parameters) and are_trivial_parameters(right): @@ -665,11 +667,22 @@ def visit_parameters(self, left: Parameters) -> bool: right = self.right if isinstance(right, CallableType): right = right.with_unpacked_kwargs() + # TODO: This heuristic sucks. We need it to make sure we normally do the strict thing. + # I wish we could just always do the strict thing.... + # This takes into advantage that these Parameters probably just came from ParamSpecType#to_parameters() + anyt = AnyType(TypeOfAny.implementation_artifact) + concat_heuristic = all( + t.arg_kinds[-2:] == [ARG_STAR, ARG_STAR2] and t.arg_names[-2:] == [None, None] and t.arg_types[-2:] == [anyt, anyt] for t in (left, right) + ) return are_parameters_compatible( left, right, is_compat=self._is_subtype, + subtype_context=self.subtype_context, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, + strict_concatenate_check=self.options.strict_concatenate + if self.options and concat_heuristic + else True, ) else: return False @@ -688,6 +701,7 @@ def visit_callable_type(self, left: CallableType) -> bool: left, right, is_compat=self._is_subtype, + subtype_context=self.subtype_context, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, strict_concatenate=self.options.strict_concatenate if self.options else True, ) @@ -722,6 +736,7 @@ def visit_callable_type(self, left: CallableType) -> bool: left.with_unpacked_kwargs(), right, is_compat=self._is_subtype, + subtype_context=self.subtype_context, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, ) else: @@ -857,6 +872,7 @@ def visit_overloaded(self, left: Overloaded) -> bool: left_item, right_item, is_compat=self._is_subtype, + subtype_context=self.subtype_context, ignore_return=True, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, strict_concatenate=strict_concat, @@ -865,6 +881,7 @@ def visit_overloaded(self, left: Overloaded) -> bool: right_item, left_item, is_compat=self._is_subtype, + subtype_context=self.subtype_context, ignore_return=True, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, strict_concatenate=strict_concat, @@ -1299,6 +1316,8 @@ def is_callable_compatible( right: CallableType, *, is_compat: Callable[[Type, Type], bool], + # TODO: should this be used more (instead of `ignore_pos_arg_names`, for instance)? + subtype_context: SubtypeContext | None = None, is_compat_return: Callable[[Type, Type], bool] | None = None, ignore_return: bool = False, ignore_pos_arg_names: bool = False, @@ -1453,6 +1472,7 @@ def g(x: int) -> int: ... left, right, is_compat=is_compat, + subtype_context=subtype_context, ignore_pos_arg_names=ignore_pos_arg_names, check_args_covariantly=check_args_covariantly, allow_partial_overlap=allow_partial_overlap, @@ -1472,11 +1492,23 @@ def are_trivial_parameters(param: Parameters | NormalizedCallableType) -> bool: ) +def are_seperated_paramspecs(param: Parameters | NormalizedCallableType) -> bool: + # TODO: based on my tests, param.arg_types will always be of length 2. This is a potential + # optimization, however Callable#param_spec() does not assume this. If this is done, update that too. + return ( + len(param.arg_types) >= 2 + and isinstance(param.arg_types[-2], ParamSpecType) + and isinstance(param.arg_types[-1], ParamSpecType) + ) + + def are_parameters_compatible( left: Parameters | NormalizedCallableType, right: Parameters | NormalizedCallableType, *, is_compat: Callable[[Type, Type], bool], + # TODO: should this be used more (instead of `ignore_pos_arg_names`, for instance)? + subtype_context: SubtypeContext | None = None, ignore_pos_arg_names: bool = False, check_args_covariantly: bool = False, allow_partial_overlap: bool = False, @@ -1495,6 +1527,19 @@ def are_parameters_compatible( if are_trivial_parameters(right): return True + # sometimes we need to compare Callable[Concatenate[prefix, P], ...] and + # (prefix, *args: P.args, **kwargs: P.kwargs) -> ... + if (isinstance(left, CallableType) and isinstance(right, CallableType)) and ( + are_seperated_paramspecs(left) or are_seperated_paramspecs(right) + ): + # note: this doesn't lose parameters. I think. Otherwise, it would be unsafe. + left_ps = left.param_spec() + right_ps = right.param_spec() + if not right_ps or not left_ps: + return False + + return is_subtype(left_ps, right_ps, subtype_context=subtype_context) + # Match up corresponding arguments and check them for compatibility. In # every pair (argL, argR) of corresponding arguments from L and R, argL must # be "more general" than argR if L is to be a subtype of R. diff --git a/mypy/types.py b/mypy/types.py index 90d33839c693..7fe31c8280e2 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -183,7 +183,7 @@ class TypeOfAny: # Does this Any come from an error? from_error: Final = 5 # Is this a type that can't be represented in mypy's type system? For instance, type of - # call to NewType...). Even though these types aren't real Anys, we treat them as such. + # call to NewType(...). Even though these types aren't real Anys, we treat them as such. # Also used for variables named '_'. special_form: Final = 6 # Does this Any come from interaction with another Any? @@ -715,6 +715,17 @@ def name_with_suffix(self) -> str: return f"{n}.kwargs" return n + def as_parameters(self) -> Parameters: + return self.prefix.copy_modified( + arg_types=self.prefix.arg_types + + [ + AnyType(TypeOfAny.implementation_artifact), + AnyType(TypeOfAny.implementation_artifact), + ], + arg_names=self.prefix.arg_names + [None, None], + arg_kinds=self.prefix.arg_kinds + [ARG_STAR, ARG_STAR2], + ) + def __hash__(self) -> int: return hash((self.id, self.flavor, self.prefix)) @@ -1970,22 +1981,19 @@ def param_spec(self) -> ParamSpecType | None: if self.arg_kinds[-2] != ARG_STAR or self.arg_kinds[-1] != ARG_STAR2: return None arg_type = self.arg_types[-2] - if not isinstance(arg_type, ParamSpecType): + if not isinstance(arg_type, ParamSpecType) or not isinstance( + self.arg_types[-1], ParamSpecType + ): return None + # sometimes paramspectypes are analyzed in from mysterious places, # e.g. def f(prefix..., *args: P.args, **kwargs: P.kwargs) -> ...: ... prefix = arg_type.prefix if not prefix.arg_types: # TODO: confirm that all arg kinds are positional prefix = Parameters(self.arg_types[:-2], self.arg_kinds[:-2], self.arg_names[:-2]) - return ParamSpecType( - arg_type.name, - arg_type.fullname, - arg_type.id, - ParamSpecFlavor.BARE, - arg_type.upper_bound, - prefix=prefix, - ) + + return arg_type.copy_modified(flavor=ParamSpecFlavor.BARE, prefix=prefix) def expand_param_spec( self, c: CallableType | Parameters, no_prefix: bool = False diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 56fc3b6faa14..3678ab770046 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1471,3 +1471,18 @@ def test(f: Concat[T, ...]) -> None: ... class Defer: ... [builtins fixtures/paramspec.pyi] + +[case testNoParamSpecDoubling] +# https://github.com/python/mypy/issues/12734 +from typing import Callable, ParamSpec +from typing_extensions import Concatenate + +P = ParamSpec("P") +Q = ParamSpec("Q") + +def foo(f: Callable[P, int]) -> Callable[P, int]: + return f + +def bar(f: Callable[Concatenate[str, Q], int]) -> Callable[Concatenate[str, Q], int]: + return foo(f) +[builtins fixtures/paramspec.pyi] From 23b3ffbb7528e42b9b58f3b50bf9f75fc4740d98 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 12 Feb 2023 08:09:57 +0900 Subject: [PATCH 02/14] Try a more minimal approach --- mypy/expandtype.py | 33 +++++++++++++++++++++++---------- mypy/messages.py | 10 ++++------ mypy/subtypes.py | 46 ---------------------------------------------- mypy/types.py | 11 ----------- 4 files changed, 27 insertions(+), 73 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 5d92c474f874..ca97af6e6ac0 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -234,15 +234,10 @@ def visit_type_var(self, t: TypeVarType) -> Type: return repl def visit_param_spec(self, t: ParamSpecType) -> Type: - repl_ = self.variables.get(t.id) - if not repl_: - # in this case, we are trying to expand it into... nothing? - # if we continue, we will get e.g.: - # repl = t = [int, str, **P] - # at which point, expand_param_spec will duplicate the arguments. - return t - - repl = get_proper_type(repl_) + # set prefix to something empty so we don't duplicate it + repl = get_proper_type( + self.variables.get(t.id, t.copy_modified(prefix=Parameters([], [], []))) + ) if isinstance(repl, Instance): # TODO: what does prefix mean in this case? # TODO: why does this case even happen? Instances aren't plural. @@ -365,7 +360,7 @@ def visit_callable_type(self, t: CallableType) -> Type: # must expand both of them with all the argument types, # kinds and names in the replacement. The return type in # the replacement is ignored. - if isinstance(repl, CallableType) or isinstance(repl, Parameters): + if isinstance(repl, (CallableType, Parameters)): # Substitute *args: P.args, **kwargs: P.kwargs prefix = param_spec.prefix # we need to expand the types in the prefix, so might as well @@ -378,6 +373,24 @@ def visit_callable_type(self, t: CallableType) -> Type: ret_type=t.ret_type.accept(self), type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), ) + # TODO: fix testConstraintsBetweenConcatenatePrefixes + # (it fails without `and repl != param_spec`) + elif isinstance(repl, ParamSpecType) and repl != param_spec: + # We're substituting one paramspec for another; this can mean that the prefix + # changes. (e.g. sub Concatenate[int, P] for Q) + prefix = repl.prefix + old_prefix = param_spec.prefix + + # Check assumptions. I'm not sure what order to switch these: + assert not old_prefix.arg_types or not prefix.arg_types + # ... and I don't know where to put non-paramspec t.arg_types + assert len(t.arg_types) == 2 or t.arg_types[:-2] == old_prefix.arg_types + + t = t.copy_modified( + arg_types=prefix.arg_types + old_prefix.arg_types + t.arg_types[-2:], + arg_kinds=prefix.arg_kinds + old_prefix.arg_kinds + t.arg_kinds[-2:], + arg_names=prefix.arg_names + old_prefix.arg_names + t.arg_names[-2:], + ) var_arg = t.var_arg() if var_arg is not None and isinstance(var_arg.typ, UnpackType): diff --git a/mypy/messages.py b/mypy/messages.py index 41adc63530fb..a5fd09493456 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2399,15 +2399,13 @@ def format_literal_value(typ: LiteralType) -> str: return_type = format(func.ret_type) if func.is_ellipsis_args: return f"Callable[..., {return_type}]" - param_spec = func.param_spec() if param_spec is not None: return f"Callable[{format(param_spec)}, {return_type}]" - else: - args = format_callable_args( - func.arg_types, func.arg_kinds, func.arg_names, format, verbosity - ) - return f"Callable[[{args}], {return_type}]" + args = format_callable_args( + func.arg_types, func.arg_kinds, func.arg_names, format, verbosity + ) + return f"Callable[[{args}], {return_type}]" else: # Use a simple representation for function types; proper # function types may result in long and difficult-to-read diff --git a/mypy/subtypes.py b/mypy/subtypes.py index fc919f498cab..41c78965b348 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -642,9 +642,6 @@ def visit_param_spec(self, left: ParamSpecType) -> bool: isinstance(right, ParamSpecType) and right.id == left.id and right.flavor == left.flavor - and is_subtype( - left.as_parameters(), right.as_parameters(), subtype_context=self.subtype_context - ) ): return True if isinstance(right, Parameters) and are_trivial_parameters(right): @@ -667,22 +664,11 @@ def visit_parameters(self, left: Parameters) -> bool: right = self.right if isinstance(right, CallableType): right = right.with_unpacked_kwargs() - # TODO: This heuristic sucks. We need it to make sure we normally do the strict thing. - # I wish we could just always do the strict thing.... - # This takes into advantage that these Parameters probably just came from ParamSpecType#to_parameters() - anyt = AnyType(TypeOfAny.implementation_artifact) - concat_heuristic = all( - t.arg_kinds[-2:] == [ARG_STAR, ARG_STAR2] and t.arg_names[-2:] == [None, None] and t.arg_types[-2:] == [anyt, anyt] for t in (left, right) - ) return are_parameters_compatible( left, right, is_compat=self._is_subtype, - subtype_context=self.subtype_context, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, - strict_concatenate_check=self.options.strict_concatenate - if self.options and concat_heuristic - else True, ) else: return False @@ -701,7 +687,6 @@ def visit_callable_type(self, left: CallableType) -> bool: left, right, is_compat=self._is_subtype, - subtype_context=self.subtype_context, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, strict_concatenate=self.options.strict_concatenate if self.options else True, ) @@ -736,7 +721,6 @@ def visit_callable_type(self, left: CallableType) -> bool: left.with_unpacked_kwargs(), right, is_compat=self._is_subtype, - subtype_context=self.subtype_context, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, ) else: @@ -872,7 +856,6 @@ def visit_overloaded(self, left: Overloaded) -> bool: left_item, right_item, is_compat=self._is_subtype, - subtype_context=self.subtype_context, ignore_return=True, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, strict_concatenate=strict_concat, @@ -881,7 +864,6 @@ def visit_overloaded(self, left: Overloaded) -> bool: right_item, left_item, is_compat=self._is_subtype, - subtype_context=self.subtype_context, ignore_return=True, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, strict_concatenate=strict_concat, @@ -1316,8 +1298,6 @@ def is_callable_compatible( right: CallableType, *, is_compat: Callable[[Type, Type], bool], - # TODO: should this be used more (instead of `ignore_pos_arg_names`, for instance)? - subtype_context: SubtypeContext | None = None, is_compat_return: Callable[[Type, Type], bool] | None = None, ignore_return: bool = False, ignore_pos_arg_names: bool = False, @@ -1472,7 +1452,6 @@ def g(x: int) -> int: ... left, right, is_compat=is_compat, - subtype_context=subtype_context, ignore_pos_arg_names=ignore_pos_arg_names, check_args_covariantly=check_args_covariantly, allow_partial_overlap=allow_partial_overlap, @@ -1492,23 +1471,11 @@ def are_trivial_parameters(param: Parameters | NormalizedCallableType) -> bool: ) -def are_seperated_paramspecs(param: Parameters | NormalizedCallableType) -> bool: - # TODO: based on my tests, param.arg_types will always be of length 2. This is a potential - # optimization, however Callable#param_spec() does not assume this. If this is done, update that too. - return ( - len(param.arg_types) >= 2 - and isinstance(param.arg_types[-2], ParamSpecType) - and isinstance(param.arg_types[-1], ParamSpecType) - ) - - def are_parameters_compatible( left: Parameters | NormalizedCallableType, right: Parameters | NormalizedCallableType, *, is_compat: Callable[[Type, Type], bool], - # TODO: should this be used more (instead of `ignore_pos_arg_names`, for instance)? - subtype_context: SubtypeContext | None = None, ignore_pos_arg_names: bool = False, check_args_covariantly: bool = False, allow_partial_overlap: bool = False, @@ -1527,19 +1494,6 @@ def are_parameters_compatible( if are_trivial_parameters(right): return True - # sometimes we need to compare Callable[Concatenate[prefix, P], ...] and - # (prefix, *args: P.args, **kwargs: P.kwargs) -> ... - if (isinstance(left, CallableType) and isinstance(right, CallableType)) and ( - are_seperated_paramspecs(left) or are_seperated_paramspecs(right) - ): - # note: this doesn't lose parameters. I think. Otherwise, it would be unsafe. - left_ps = left.param_spec() - right_ps = right.param_spec() - if not right_ps or not left_ps: - return False - - return is_subtype(left_ps, right_ps, subtype_context=subtype_context) - # Match up corresponding arguments and check them for compatibility. In # every pair (argL, argR) of corresponding arguments from L and R, argL must # be "more general" than argR if L is to be a subtype of R. diff --git a/mypy/types.py b/mypy/types.py index 7fe31c8280e2..783eb28a2912 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -715,17 +715,6 @@ def name_with_suffix(self) -> str: return f"{n}.kwargs" return n - def as_parameters(self) -> Parameters: - return self.prefix.copy_modified( - arg_types=self.prefix.arg_types - + [ - AnyType(TypeOfAny.implementation_artifact), - AnyType(TypeOfAny.implementation_artifact), - ], - arg_names=self.prefix.arg_names + [None, None], - arg_kinds=self.prefix.arg_kinds + [ARG_STAR, ARG_STAR2], - ) - def __hash__(self) -> int: return hash((self.id, self.flavor, self.prefix)) From 2d141f1bf760ef79672ece06ac522f93dc3df2fb Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 12 Feb 2023 09:01:12 +0900 Subject: [PATCH 03/14] Fix crash in discord.py --- mypy/expandtype.py | 15 ++++++------- .../unit/check-parameter-specification.test | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ca97af6e6ac0..df375ac1d27c 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -373,9 +373,10 @@ def visit_callable_type(self, t: CallableType) -> Type: ret_type=t.ret_type.accept(self), type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), ) - # TODO: fix testConstraintsBetweenConcatenatePrefixes - # (it fails without `and repl != param_spec`) - elif isinstance(repl, ParamSpecType) and repl != param_spec: + # TODO: it seems this only has to be done *sometimes*. Conceptually this should only + # be done once; we should update that "once" location rather than here. + # (see testAlreadyExpandedCallableWithParamSpecReplacement) + elif isinstance(repl, ParamSpecType) and len(t.arg_types) == 2: # We're substituting one paramspec for another; this can mean that the prefix # changes. (e.g. sub Concatenate[int, P] for Q) prefix = repl.prefix @@ -383,13 +384,11 @@ def visit_callable_type(self, t: CallableType) -> Type: # Check assumptions. I'm not sure what order to switch these: assert not old_prefix.arg_types or not prefix.arg_types - # ... and I don't know where to put non-paramspec t.arg_types - assert len(t.arg_types) == 2 or t.arg_types[:-2] == old_prefix.arg_types t = t.copy_modified( - arg_types=prefix.arg_types + old_prefix.arg_types + t.arg_types[-2:], - arg_kinds=prefix.arg_kinds + old_prefix.arg_kinds + t.arg_kinds[-2:], - arg_names=prefix.arg_names + old_prefix.arg_names + t.arg_names[-2:], + arg_types=prefix.arg_types + old_prefix.arg_types + t.arg_types, + arg_kinds=prefix.arg_kinds + old_prefix.arg_kinds + t.arg_kinds, + arg_names=prefix.arg_names + old_prefix.arg_names + t.arg_names, ) var_arg = t.var_arg() diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 3678ab770046..2255d315358a 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1486,3 +1486,25 @@ def foo(f: Callable[P, int]) -> Callable[P, int]: def bar(f: Callable[Concatenate[str, Q], int]) -> Callable[Concatenate[str, Q], int]: return foo(f) [builtins fixtures/paramspec.pyi] + +[case testAlreadyExpandedCallableWithParamSpecReplacement] +from typing import Callable, Any, overload +from typing_extensions import Concatenate, ParamSpec + +P = ParamSpec("P") + +@overload +def command() -> Callable[[Callable[Concatenate[object, object, P], object]], None]: + ... + +@overload +def command( + cls: int = ..., +) -> Callable[[Callable[Concatenate[object, P], object]], None]: + ... + +def command( + cls: int = 42, +) -> Any: + ... +[builtins fixtures/paramspec.pyi] From d58f194ad1ffb6ba247d7bc74a26d43a2f50c6df Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 12 Feb 2023 09:07:32 +0900 Subject: [PATCH 04/14] Include errors on the regression test --- test-data/unit/check-parameter-specification.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 2255d315358a..9c8f1699d05d 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1494,7 +1494,7 @@ from typing_extensions import Concatenate, ParamSpec P = ParamSpec("P") @overload -def command() -> Callable[[Callable[Concatenate[object, object, P], object]], None]: +def command() -> Callable[[Callable[Concatenate[object, object, P], object]], None]: # E: Overloaded function signatures 1 and 2 overlap with incompatible return types ... @overload From ed520341197d313fa9bab76cea126e1c65776d37 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 20 Feb 2023 16:11:48 +0900 Subject: [PATCH 05/14] PR feedback --- mypy/expandtype.py | 8 ++++---- mypy/types.py | 4 +--- test-data/unit/check-parameter-specification.test | 13 +++++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index df375ac1d27c..d22bc6cd0b51 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -373,16 +373,16 @@ def visit_callable_type(self, t: CallableType) -> Type: ret_type=t.ret_type.accept(self), type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), ) - # TODO: it seems this only has to be done *sometimes*. Conceptually this should only - # be done once; we should update that "once" location rather than here. - # (see testAlreadyExpandedCallableWithParamSpecReplacement) + # TODO: Conceptually, the "len(t.arg_types) == 2" should not be here. However, this + # errors without it. Either figure out how to eliminate this or place an + # explanation for why this is necessary. elif isinstance(repl, ParamSpecType) and len(t.arg_types) == 2: # We're substituting one paramspec for another; this can mean that the prefix # changes. (e.g. sub Concatenate[int, P] for Q) prefix = repl.prefix old_prefix = param_spec.prefix - # Check assumptions. I'm not sure what order to switch these: + # Check assumptions. I'm not sure what order to place new prefix vs old prefix: assert not old_prefix.arg_types or not prefix.arg_types t = t.copy_modified( diff --git a/mypy/types.py b/mypy/types.py index 783eb28a2912..964da07c56cb 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1970,9 +1970,7 @@ def param_spec(self) -> ParamSpecType | None: if self.arg_kinds[-2] != ARG_STAR or self.arg_kinds[-1] != ARG_STAR2: return None arg_type = self.arg_types[-2] - if not isinstance(arg_type, ParamSpecType) or not isinstance( - self.arg_types[-1], ParamSpecType - ): + if not isinstance(arg_type, ParamSpecType): return None # sometimes paramspectypes are analyzed in from mysterious places, diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 9c8f1699d05d..7ef0485f7841 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1508,3 +1508,16 @@ def command( ) -> Any: ... [builtins fixtures/paramspec.pyi] + +[case testCopiedParamSpecComparison] +# minimized from https://github.com/python/mypy/issues/12909 +from typing import Callable +from typing_extensions import ParamSpec + +P = ParamSpec("P") + +def identity(func: Callable[P, None]) -> Callable[P, None]: ... + +@identity +def f(f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... +[builtins fixtures/paramspec.pyi] From 24f4e2db06416288a82f6e793213b7c926377b1e Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 22 Feb 2023 10:07:28 +0900 Subject: [PATCH 06/14] Fix strange constraint behavior for ParamSpec Note that I'm not too sure the comment is correct. --- mypy/constraints.py | 50 +++++++++++++++---- .../unit/check-parameter-specification.test | 24 +++++++++ 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index c8c3c7933b6e..b8ef7388864f 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -176,12 +176,13 @@ def infer_constraints_for_callable( def infer_constraints(template: Type, actual: Type, direction: int) -> list[Constraint]: """Infer type constraints. - Match a template type, which may contain type variable references, - recursively against a type which does not contain (the same) type - variable references. The result is a list of type constrains of - form 'T is a supertype/subtype of x', where T is a type variable - present in the template and x is a type without reference to type - variables present in the template. + Match a template type, which may contain type variable and parameter + specification references, recursively against a type which does not + contain (the same) type variable and parameter specification references. + The result is a list of type constraints of form 'T is a supertype/subtype + of x', where T is a type variable present in the template or a parameter + specification without its prefix and x is a type without reference to type + variables nor parameters present in the template. Assume T and S are type variables. Now the following results can be calculated (read as '(template, actual) --> result'): @@ -192,6 +193,23 @@ def infer_constraints(template: Type, actual: Type, direction: int) -> list[Cons ((T, S), (X, Y)) --> T :> X and S :> Y (X[T], Any) --> T <: Any and T :> Any + Assume P and Q are prefix-less parameter specifications. The following + results can be calculated in a similar format: + + (P, [...W]) --> P :> [...W] + (X[P], X[[...W]]) --> P :> [...W] + // note that parameter specifications are *always* contravariant as + // they echo Callable arguments. + ((P, P), ([...W], [...U])) --> P :> [...W] and P :> [...U] + ((P, Q), ([...W], [...U])) --> P :> [...W] and Q :> [...U] + (P, ...) --> P :> ... + + With prefixes (note that I am not sure these cases are implemented): + + ([...Z, P], [...Z, ...W]) --> P :> [...W] + ([...Z, P], Q) --> [...Z, P] :> Q + (P, [...Z, Q]) --> P :> [...Z, Q] + The constraints are represented as Constraint objects. """ if any( @@ -918,14 +936,20 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # sometimes, it appears we try to get constraints between two paramspec callables? # TODO: Direction - # TODO: check the prefixes match prefix = param_spec.prefix prefix_len = len(prefix.arg_types) cactual_ps = cactual.param_spec() + if cactual_ps: + cactual_prefix = cactual_ps.prefix + else: + cactual_prefix = cactual + + max_prefix_len = len([k for k in cactual_prefix.arg_kinds if k in (ARG_POS, ARG_OPT)]) + prefix_len = min(prefix_len, max_prefix_len) + + # we could check the prefixes match here, but that should be caught elsewhere. if not cactual_ps: - max_prefix_len = len([k for k in cactual.arg_kinds if k in (ARG_POS, ARG_OPT)]) - prefix_len = min(prefix_len, max_prefix_len) res.append( Constraint( param_spec, @@ -939,7 +963,13 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: ) ) else: - res.append(Constraint(param_spec, SUBTYPE_OF, cactual_ps)) + res.append(Constraint(param_spec, SUBTYPE_OF, cactual_ps.copy_modified( + prefix=cactual_prefix.copy_modified( + arg_types=cactual_prefix.arg_types[prefix_len:], + arg_kinds=cactual_prefix.arg_kinds[prefix_len:], + arg_names=cactual_prefix.arg_names[prefix_len:] + ) + ))) # compare prefixes cactual_prefix = cactual.copy_modified( diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 7ef0485f7841..e994477c3012 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1521,3 +1521,27 @@ def identity(func: Callable[P, None]) -> Callable[P, None]: ... @identity def f(f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... [builtins fixtures/paramspec.pyi] + +[case testRemoveSharedPrefixForConstraining] +# copied essentially verbatim from testing usages of ParamSpec +from typing import TypeVar, Callable +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +T = TypeVar("T") +V = TypeVar("V") +R = TypeVar("R") + +def command_builder() -> Callable[[Callable[Concatenate[T, P], R]], Callable[P, Callable[[T], R]]]: + def transformer(f: Callable[Concatenate[T, P], R], /) -> Callable[P, Callable[[T], R]]: + def returned(*args: P.args, **kwargs: P.kwargs) -> Callable[[T], R]: + def returned_transformer(z: T) -> R: + return f(z, *args, **kwargs) + + return returned_transformer + + return returned + + reveal_type(transformer) # N: Revealed type is "def [T, P, R] (def (T`-1, *P.args, **P.kwargs) -> R`-3) -> def (*P.args, **P.kwargs) -> def (T`-1) -> R`-3" + return transformer +[builtins fixtures/paramspec.pyi] From 9f084c09dba6277a0549b7d64cc543967d5cf7c5 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Thu, 23 Feb 2023 06:43:04 +0900 Subject: [PATCH 07/14] Fix some easy-to-spot bugs --- mypy/constraints.py | 25 ++++++++++++++++--------- mypy/erasetype.py | 6 +++++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index b8ef7388864f..0c0bc942508c 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -713,19 +713,26 @@ def visit_instance(self, template: Instance) -> list[Constraint]: from_concat = bool(prefix.arg_types) or suffix.from_concatenate suffix = suffix.copy_modified(from_concatenate=from_concat) + + prefix = mapped_arg.prefix + length = len(prefix.arg_types) if isinstance(suffix, Parameters) or isinstance(suffix, CallableType): # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? - # TODO: constraints between prefixes - prefix = mapped_arg.prefix - suffix = suffix.copy_modified( - suffix.arg_types[len(prefix.arg_types) :], - suffix.arg_kinds[len(prefix.arg_kinds) :], - suffix.arg_names[len(prefix.arg_names) :], - ) - res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix)) + res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix.copy_modified( + arg_types=suffix.arg_types[length:], + arg_kinds=suffix.arg_kinds[length:], + arg_names=suffix.arg_names[length:], + ))) elif isinstance(suffix, ParamSpecType): - res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix)) + suffix_prefix = suffix.prefix + res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix.copy_modified( + prefix=suffix_prefix.copy_modified( + arg_types=suffix_prefix.arg_types[length:], + arg_kinds=suffix_prefix.arg_kinds[length:], + arg_names=suffix_prefix.arg_names[length:] + ) + ))) else: # This case should have been handled above. assert not isinstance(tvar, TypeVarTupleType) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 6533d0c4e0f9..e9dacf517a8a 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -172,7 +172,11 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: def visit_param_spec(self, t: ParamSpecType) -> Type: if self.erase_id(t.id): - return self.replacement + return t.prefix.copy_modified( + arg_types=t.prefix.arg_types + [self.replacement, self.replacement], + arg_kinds=t.prefix.arg_kinds + [ARG_STAR, ARG_STAR2], + arg_names=t.prefix.arg_names + [None, None] + ) return t def visit_type_alias_type(self, t: TypeAliasType) -> Type: From 86993a0f77976e0cbbbec4456797d8bc2abccb9d Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 25 Feb 2023 09:56:27 +0900 Subject: [PATCH 08/14] Some other fixes --- mypy/join.py | 6 ++++-- mypy/nodes.py | 7 +++++++ mypy/semanal.py | 9 ++++++++- mypy/subtypes.py | 2 ++ mypy/types.py | 19 ++++++++++++++++--- .../unit/check-parameter-specification.test | 19 ++++++++++++++++--- 6 files changed, 53 insertions(+), 9 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index 62d256f4440f..ec0ba865367c 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -97,7 +97,7 @@ def join_instances(self, t: Instance, s: Instance) -> ProperType: else: # ParamSpec type variables behave the same, independent of variance if not is_equivalent(ta, sa): - return get_proper_type(type_var.upper_bound) + return object_from_instance(t) new_type = join_types(ta, sa, self) assert new_type is not None args.append(new_type) @@ -311,9 +311,11 @@ def visit_type_var(self, t: TypeVarType) -> ProperType: return self.default(self.s) def visit_param_spec(self, t: ParamSpecType) -> ProperType: + # TODO: should this mirror the `isinstance(...) ...` above? if self.s == t: return t - return self.default(self.s) + else: + return self.default(self.s) def visit_type_var_tuple(self, t: TypeVarTupleType) -> ProperType: if self.s == t: diff --git a/mypy/nodes.py b/mypy/nodes.py index 4787930214f3..ac0c98776ce1 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2510,6 +2510,13 @@ class ParamSpecExpr(TypeVarLikeExpr): __match_args__ = ("name", "upper_bound") + # TODO: Technically the variance cannot be customized. Nor can the upper bound. + def __init__( + self, name: str, fullname: str, upper_bound: mypy.types.Type, variance: int = INVARIANT + ) -> None: + super().__init__(name, fullname, upper_bound, variance) + assert isinstance(upper_bound, (mypy.types.CallableType, mypy.types.Parameters)) + def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_paramspec_expr(self) diff --git a/mypy/semanal.py b/mypy/semanal.py index d2fd92499679..599c037a0cfe 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4160,7 +4160,7 @@ def process_paramspec_declaration(self, s: AssignmentStmt) -> bool: if not call.analyzed: paramspec_var = ParamSpecExpr( - name, self.qualified_name(name), self.object_type(), INVARIANT + name, self.qualified_name(name), self.top_caller(), INVARIANT ) paramspec_var.line = call.line call.analyzed = paramspec_var @@ -5602,6 +5602,13 @@ def lookup_fully_qualified_or_none(self, fullname: str) -> SymbolTableNode | Non def object_type(self) -> Instance: return self.named_type("builtins.object") + def top_caller(self) -> Parameters: + return Parameters( + arg_types=[self.object_type(), self.object_type()], + arg_kinds=[ARG_STAR, ARG_STAR2], + arg_names=[None, None] + ) + def str_type(self) -> Instance: return self.named_type("builtins.str") diff --git a/mypy/subtypes.py b/mypy/subtypes.py index b322cf7b6cd8..676c2c75366f 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -592,6 +592,8 @@ def check_mixed( ): nominal = False else: + # TODO: I'm *pretty* sure `CONTRAVARIANT` should be here... + # But it's erroring! if not check_type_parameter( lefta, righta, COVARIANT, self.proper_subtype, self.subtype_context ): diff --git a/mypy/types.py b/mypy/types.py index c21e3f5f7bed..6c99524f7dea 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -651,7 +651,8 @@ class ParamSpecType(TypeVarLikeType): The upper_bound is really used as a fallback type -- it's shared with TypeVarType for simplicity. It can't be specified by the user and the value is directly derived from the flavor (currently - always just 'object'). + always just '(*Any, **Any)' or '(*object, **object)' depending on + context). """ __slots__ = ("flavor", "prefix") @@ -674,6 +675,7 @@ def __init__( super().__init__(name, fullname, id, upper_bound, line=line, column=column) self.flavor = flavor self.prefix = prefix or Parameters([], [], []) + assert flavor != ParamSpecFlavor.BARE or isinstance(upper_bound, (CallableType, Parameters)) @staticmethod def new_unification_variable(old: ParamSpecType) -> ParamSpecType: @@ -696,13 +698,14 @@ def copy_modified( id: Bogus[TypeVarId | int] = _dummy, flavor: int = _dummy_int, prefix: Bogus[Parameters] = _dummy, + upper_bound: Bogus[Type] = _dummy, ) -> ParamSpecType: return ParamSpecType( self.name, self.fullname, id if id is not _dummy else self.id, flavor if flavor != _dummy_int else self.flavor, - self.upper_bound, + upper_bound if upper_bound is not _dummy else self.upper_bound, line=self.line, column=self.column, prefix=prefix if prefix is not _dummy else self.prefix, @@ -1984,7 +1987,17 @@ def param_spec(self) -> ParamSpecType | None: # TODO: confirm that all arg kinds are positional prefix = Parameters(self.arg_types[:-2], self.arg_kinds[:-2], self.arg_names[:-2]) - return arg_type.copy_modified(flavor=ParamSpecFlavor.BARE, prefix=prefix) + # TODO: should this take in `object`s? + any_type = AnyType(TypeOfAny.special_form) + return arg_type.copy_modified( + flavor=ParamSpecFlavor.BARE, + prefix=prefix, + upper_bound=Parameters( + arg_types=[any_type, any_type], + arg_kinds=[ARG_STAR, ARG_STAR2], + arg_names=[None, None] + ) + ) def expand_param_spec( self, c: CallableType | Parameters, no_prefix: bool = False diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index e994477c3012..0003f7c6e56d 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1524,7 +1524,7 @@ def f(f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... [case testRemoveSharedPrefixForConstraining] # copied essentially verbatim from testing usages of ParamSpec -from typing import TypeVar, Callable +from typing import TypeVar, Callable, Generic from typing_extensions import ParamSpec, Concatenate P = ParamSpec("P") @@ -1533,7 +1533,7 @@ V = TypeVar("V") R = TypeVar("R") def command_builder() -> Callable[[Callable[Concatenate[T, P], R]], Callable[P, Callable[[T], R]]]: - def transformer(f: Callable[Concatenate[T, P], R], /) -> Callable[P, Callable[[T], R]]: + def transformer(f: Callable[Concatenate[T, P], R]) -> Callable[P, Callable[[T], R]]: def returned(*args: P.args, **kwargs: P.kwargs) -> Callable[[T], R]: def returned_transformer(z: T) -> R: return f(z, *args, **kwargs) @@ -1542,6 +1542,19 @@ def command_builder() -> Callable[[Callable[Concatenate[T, P], R]], Callable[P, return returned - reveal_type(transformer) # N: Revealed type is "def [T, P, R] (def (T`-1, *P.args, **P.kwargs) -> R`-3) -> def (*P.args, **P.kwargs) -> def (T`-1) -> R`-3" + reveal_type(transformer) # N: Revealed type is "def [T, P, R] (f: def (T`-1, *P.args, **P.kwargs) -> R`-3) -> def (*P.args, **P.kwargs) -> def (T`-1) -> R`-3" return transformer + +class Example(Generic[P]): + pass + +@command_builder() +def test(ex: Example[P]) -> Example[Concatenate[int, P]]: + ... + +ex: Example[int] = test()(reveal_type(Example())) # N: Revealed type is "__main__.Example[]" +# TODO: fix +reveal_type(test()(Example[int]())) # N: Revealed type is "__main__.Example[]" \ + # E: Argument 1 has incompatible type "Example[[int]]"; expected "Example[]" +ex = test()(Example[int]()) # E: Argument 1 has incompatible type "Example[[int]]"; expected "Example[]" [builtins fixtures/paramspec.pyi] From 8642f3dede9ffc48ad4eae28e9dd46a96567ed63 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 25 Feb 2023 15:30:47 +0900 Subject: [PATCH 09/14] Fix inference bug --- mypy/checkexpr.py | 20 ++++++ mypy/constraints.py | 63 +++++++++++++------ mypy/erasetype.py | 2 +- mypy/expandtype.py | 1 - mypy/nodes.py | 1 - mypy/semanal.py | 2 +- mypy/strconv.py | 5 +- mypy/types.py | 5 +- test-data/unit/check-inference.test | 5 +- .../unit/check-parameter-specification.test | 8 +-- 10 files changed, 77 insertions(+), 35 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 38b5c2419d95..787ee29286ff 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1776,6 +1776,26 @@ def infer_function_type_arguments( callee_type, args, arg_kinds, formal_to_actual, inferred_args, context ) + return_type = get_proper_type(callee_type.ret_type) + if isinstance(return_type, CallableType): + # fixup: + # def [T] () -> def (T) -> T + # into + # def () -> def [T] (T) -> T + for i, argument in enumerate(inferred_args): + if isinstance(get_proper_type(argument), UninhabitedType): + inferred_args[i] = callee_type.variables[i] + + # handle multiple type variables + return_type = return_type.copy_modified( + variables=[*return_type.variables, callee_type.variables[i]] + ) + + callee_type = callee_type.copy_modified( + # am I allowed to assign the get_proper_type'd thing? + ret_type=return_type + ) + if ( callee_type.special_sig == "dict" and len(inferred_args) == 2 diff --git a/mypy/constraints.py b/mypy/constraints.py index 0c0bc942508c..689d294b9ef6 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, List, Sequence +from typing import TYPE_CHECKING, Iterable, List, Sequence, Union from typing_extensions import Final import mypy.subtypes @@ -713,26 +713,37 @@ def visit_instance(self, template: Instance) -> list[Constraint]: from_concat = bool(prefix.arg_types) or suffix.from_concatenate suffix = suffix.copy_modified(from_concatenate=from_concat) - prefix = mapped_arg.prefix length = len(prefix.arg_types) if isinstance(suffix, Parameters) or isinstance(suffix, CallableType): # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? - res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix.copy_modified( - arg_types=suffix.arg_types[length:], - arg_kinds=suffix.arg_kinds[length:], - arg_names=suffix.arg_names[length:], - ))) + res.append( + Constraint( + mapped_arg, + SUPERTYPE_OF, + suffix.copy_modified( + arg_types=suffix.arg_types[length:], + arg_kinds=suffix.arg_kinds[length:], + arg_names=suffix.arg_names[length:], + ), + ) + ) elif isinstance(suffix, ParamSpecType): suffix_prefix = suffix.prefix - res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix.copy_modified( - prefix=suffix_prefix.copy_modified( - arg_types=suffix_prefix.arg_types[length:], - arg_kinds=suffix_prefix.arg_kinds[length:], - arg_names=suffix_prefix.arg_names[length:] + res.append( + Constraint( + mapped_arg, + SUPERTYPE_OF, + suffix.copy_modified( + prefix=suffix_prefix.copy_modified( + arg_types=suffix_prefix.arg_types[length:], + arg_kinds=suffix_prefix.arg_kinds[length:], + arg_names=suffix_prefix.arg_names[length:], + ) + ), ) - ))) + ) else: # This case should have been handled above. assert not isinstance(tvar, TypeVarTupleType) @@ -947,12 +958,15 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: prefix_len = len(prefix.arg_types) cactual_ps = cactual.param_spec() + cactual_prefix: Union[Parameters, CallableType] if cactual_ps: cactual_prefix = cactual_ps.prefix else: cactual_prefix = cactual - max_prefix_len = len([k for k in cactual_prefix.arg_kinds if k in (ARG_POS, ARG_OPT)]) + max_prefix_len = len( + [k for k in cactual_prefix.arg_kinds if k in (ARG_POS, ARG_OPT)] + ) prefix_len = min(prefix_len, max_prefix_len) # we could check the prefixes match here, but that should be caught elsewhere. @@ -970,13 +984,22 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: ) ) else: - res.append(Constraint(param_spec, SUBTYPE_OF, cactual_ps.copy_modified( - prefix=cactual_prefix.copy_modified( - arg_types=cactual_prefix.arg_types[prefix_len:], - arg_kinds=cactual_prefix.arg_kinds[prefix_len:], - arg_names=cactual_prefix.arg_names[prefix_len:] + # guaranteed due to if conditions + assert isinstance(cactual_prefix, Parameters) + + res.append( + Constraint( + param_spec, + SUBTYPE_OF, + cactual_ps.copy_modified( + prefix=cactual_prefix.copy_modified( + arg_types=cactual_prefix.arg_types[prefix_len:], + arg_kinds=cactual_prefix.arg_kinds[prefix_len:], + arg_names=cactual_prefix.arg_names[prefix_len:], + ) + ), ) - ))) + ) # compare prefixes cactual_prefix = cactual.copy_modified( diff --git a/mypy/erasetype.py b/mypy/erasetype.py index e9dacf517a8a..43dbe3a0d8e4 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -175,7 +175,7 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: return t.prefix.copy_modified( arg_types=t.prefix.arg_types + [self.replacement, self.replacement], arg_kinds=t.prefix.arg_kinds + [ARG_STAR, ARG_STAR2], - arg_names=t.prefix.arg_names + [None, None] + arg_names=t.prefix.arg_names + [None, None], ) return t diff --git a/mypy/expandtype.py b/mypy/expandtype.py index d22bc6cd0b51..a9a6a03d78d5 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -126,7 +126,6 @@ def freshen_function_type_vars(callee: F) -> F: if isinstance(v, TypeVarType): tv: TypeVarLikeType = TypeVarType.new_unification_variable(v) elif isinstance(v, TypeVarTupleType): - assert isinstance(v, TypeVarTupleType) tv = TypeVarTupleType.new_unification_variable(v) else: assert isinstance(v, ParamSpecType) diff --git a/mypy/nodes.py b/mypy/nodes.py index ac0c98776ce1..c5416bfc8be9 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2515,7 +2515,6 @@ def __init__( self, name: str, fullname: str, upper_bound: mypy.types.Type, variance: int = INVARIANT ) -> None: super().__init__(name, fullname, upper_bound, variance) - assert isinstance(upper_bound, (mypy.types.CallableType, mypy.types.Parameters)) def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_paramspec_expr(self) diff --git a/mypy/semanal.py b/mypy/semanal.py index 599c037a0cfe..20f42f020877 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5606,7 +5606,7 @@ def top_caller(self) -> Parameters: return Parameters( arg_types=[self.object_type(), self.object_type()], arg_kinds=[ARG_STAR, ARG_STAR2], - arg_names=[None, None] + arg_names=[None, None], ) def str_type(self) -> Instance: diff --git a/mypy/strconv.py b/mypy/strconv.py index b2e9da5dbf6a..2fad788cb9f3 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -484,8 +484,9 @@ def visit_paramspec_expr(self, o: mypy.nodes.ParamSpecExpr) -> str: a += ["Variance(COVARIANT)"] if o.variance == mypy.nodes.CONTRAVARIANT: a += ["Variance(CONTRAVARIANT)"] - if not mypy.types.is_named_instance(o.upper_bound, "builtins.object"): - a += [f"UpperBound({o.upper_bound})"] + # ParamSpecs do not have upper bounds!!! (should this be left for future proofing?) + # if not mypy.types.is_named_instance(o.upper_bound, "builtins.object"): + # a += [f"UpperBound({o.upper_bound})"] return self.dump(a, o) def visit_type_var_tuple_expr(self, o: mypy.nodes.TypeVarTupleExpr) -> str: diff --git a/mypy/types.py b/mypy/types.py index 6c99524f7dea..8d80acafa714 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -675,7 +675,6 @@ def __init__( super().__init__(name, fullname, id, upper_bound, line=line, column=column) self.flavor = flavor self.prefix = prefix or Parameters([], [], []) - assert flavor != ParamSpecFlavor.BARE or isinstance(upper_bound, (CallableType, Parameters)) @staticmethod def new_unification_variable(old: ParamSpecType) -> ParamSpecType: @@ -1995,8 +1994,8 @@ def param_spec(self) -> ParamSpecType | None: upper_bound=Parameters( arg_types=[any_type, any_type], arg_kinds=[ARG_STAR, ARG_STAR2], - arg_names=[None, None] - ) + arg_names=[None, None], + ), ) def expand_param_spec( diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index fc8113766f1a..be08f29542d7 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -2956,8 +2956,11 @@ T = TypeVar('T') def f(x: Optional[T] = None) -> Callable[..., T]: ... -x = f() # E: Need type annotation for "x" +# TODO: should this warn about needed an annotation? This behavior still _works_... +x = f() +reveal_type(x) # N: Revealed type is "def [T] (*Any, **Any) -> T`1" y = x +reveal_type(y) # N: Revealed type is "def [T] (*Any, **Any) -> T`1" [case testDontNeedAnnotationForCallable] from typing import TypeVar, Optional, Callable, NoReturn diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 0003f7c6e56d..1f7d8a2823f3 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1552,9 +1552,7 @@ class Example(Generic[P]): def test(ex: Example[P]) -> Example[Concatenate[int, P]]: ... -ex: Example[int] = test()(reveal_type(Example())) # N: Revealed type is "__main__.Example[]" -# TODO: fix -reveal_type(test()(Example[int]())) # N: Revealed type is "__main__.Example[]" \ - # E: Argument 1 has incompatible type "Example[[int]]"; expected "Example[]" -ex = test()(Example[int]()) # E: Argument 1 has incompatible type "Example[[int]]"; expected "Example[]" +ex: Example[int] = test()(reveal_type(Example())) # N: Revealed type is "__main__.Example[[]]" +reveal_type(test()(Example[int]())) # N: Revealed type is "__main__.Example[[builtins.int, builtins.int]]" +ex = test()(Example[int]()) # E: Argument 1 has incompatible type "Example[[int]]"; expected "Example[[]]" [builtins fixtures/paramspec.pyi] From aad6a86e5b6e2d1b6a448776936f54de19c7e2ab Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 25 Mar 2023 10:37:42 +0900 Subject: [PATCH 10/14] Skip unification variables --- mypy/checkexpr.py | 7 ++++++- mypy/semanal.py | 1 + mypy/types.py | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 6a3f7ebc65a7..70187026e7b3 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1783,7 +1783,12 @@ def infer_function_type_arguments( # into # def () -> def [T] (T) -> T for i, argument in enumerate(inferred_args): - if isinstance(get_proper_type(argument), UninhabitedType): + # NOTE: I'm not too sure about my concept of meta variables. Maybe we + # shouldn't skip them, but instead make them non-meta? + if ( + isinstance(get_proper_type(argument), UninhabitedType) + and not callee_type.variables[i].id.is_meta_var() + ): inferred_args[i] = callee_type.variables[i] # handle multiple type variables diff --git a/mypy/semanal.py b/mypy/semanal.py index 190bee03e549..850122244941 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5617,6 +5617,7 @@ def top_caller(self) -> Parameters: arg_types=[self.object_type(), self.object_type()], arg_kinds=[ARG_STAR, ARG_STAR2], arg_names=[None, None], + is_ellipsis_args=True, ) def str_type(self) -> Instance: diff --git a/mypy/types.py b/mypy/types.py index b08c5706d87c..7e18e54e2b52 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1997,6 +1997,7 @@ def param_spec(self) -> ParamSpecType | None: arg_types=[any_type, any_type], arg_kinds=[ARG_STAR, ARG_STAR2], arg_names=[None, None], + is_ellipsis_args=True, ), ) From b3b0d2297e0fa56bba5e0bcc501bdfcab277735b Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 25 Mar 2023 11:15:19 +0900 Subject: [PATCH 11/14] Instead of skipping, try un-unifying them I don't actually know what I'm doing. The tests pass, I am happy. --- mypy/checkexpr.py | 17 ++++++++--------- test-data/unit/check-inference.test | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 70187026e7b3..faaff2a04967 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1429,6 +1429,7 @@ def check_callable_call( need_refresh = any( isinstance(v, (ParamSpecType, TypeVarTupleType)) for v in callee.variables ) + old_callee = callee callee = freshen_function_type_vars(callee) callee = self.infer_function_type_arguments_using_context(callee, context) if need_refresh: @@ -1443,7 +1444,7 @@ def check_callable_call( lambda i: self.accept(args[i]), ) callee = self.infer_function_type_arguments( - callee, args, arg_kinds, formal_to_actual, context + callee, args, arg_kinds, formal_to_actual, context, old_callee ) if need_refresh: formal_to_actual = map_actuals_to_formals( @@ -1733,6 +1734,7 @@ def infer_function_type_arguments( arg_kinds: list[ArgKind], formal_to_actual: list[list[int]], context: Context, + unfreshened_callee_type: CallableType, ) -> CallableType: """Infer the type arguments for a generic callee type. @@ -1783,17 +1785,14 @@ def infer_function_type_arguments( # into # def () -> def [T] (T) -> T for i, argument in enumerate(inferred_args): - # NOTE: I'm not too sure about my concept of meta variables. Maybe we - # shouldn't skip them, but instead make them non-meta? - if ( - isinstance(get_proper_type(argument), UninhabitedType) - and not callee_type.variables[i].id.is_meta_var() - ): - inferred_args[i] = callee_type.variables[i] + if isinstance(get_proper_type(argument), UninhabitedType): + # un-"freshen" the type variable :^) + variable = unfreshened_callee_type.variables[i] + inferred_args[i] = variable # handle multiple type variables return_type = return_type.copy_modified( - variables=[*return_type.variables, callee_type.variables[i]] + variables=[*return_type.variables, variable] ) callee_type = callee_type.copy_modified( diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index e3180be83d98..757e5205762e 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -3105,9 +3105,9 @@ def f(x: Optional[T] = None) -> Callable[..., T]: ... # TODO: should this warn about needed an annotation? This behavior still _works_... x = f() -reveal_type(x) # N: Revealed type is "def [T] (*Any, **Any) -> T`1" +reveal_type(x) # N: Revealed type is "def [T] (*Any, **Any) -> T`-1" y = x -reveal_type(y) # N: Revealed type is "def [T] (*Any, **Any) -> T`1" +reveal_type(y) # N: Revealed type is "def [T] (*Any, **Any) -> T`-1" [case testDontNeedAnnotationForCallable] from typing import TypeVar, Optional, Callable, NoReturn From 1bc9b75669bafea6536d9c68362181976fb53fcc Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 15 Apr 2023 00:52:08 +0900 Subject: [PATCH 12/14] Fix runtime paramspec literal parsing for the special syntax (Ex[...] not Ex[[...]]) --- mypy/checkexpr.py | 6 ++++++ mypy/typeanal.py | 3 +++ test-data/unit/check-parameter-specification.test | 13 +++++++++++++ 3 files changed, 22 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index faaff2a04967..c8bcce959d1e 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -136,6 +136,7 @@ LiteralValue, NoneType, Overloaded, + Parameters, ParamSpecFlavor, ParamSpecType, PartialType, @@ -4095,6 +4096,11 @@ def apply_type_arguments_to_callable( tp = get_proper_type(tp) if isinstance(tp, CallableType): + if len(tp.variables) == 1 and isinstance(tp.variables[0], ParamSpecType) and (len(args) != 1 or not isinstance(args[0], (Parameters, ParamSpecType, AnyType))): + # TODO: I don't think AnyType here is valid in the general case, there's 2 cases: + # 1. invalid paramspec expression (in which case we should transform it into an ellipsis) + # 2. user passed it (in which case we should pass it into Parameters(...)) + args = [Parameters(args, [nodes.ARG_POS for _ in args], [None for _ in args])] if len(tp.variables) != len(args): if tp.is_type_obj() and tp.type_object().fullname == "builtins.tuple": # TODO: Specialize the callable for the type arguments diff --git a/mypy/typeanal.py b/mypy/typeanal.py index f3329af6207a..63773da656fa 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -447,6 +447,9 @@ def pack_paramspec_args(self, an_args: Sequence[Type]) -> list[Type]: if count > 0: first_arg = get_proper_type(an_args[0]) if not (count == 1 and isinstance(first_arg, (Parameters, ParamSpecType, AnyType))): + # TODO: I don't think AnyType here is valid in the general case, there's 2 cases: + # 1. invalid paramspec expression (in which case we should transform it into an ellipsis) + # 2. user passed it (in which case we should pass it into Parameters(...)) return [Parameters(an_args, [ARG_POS] * count, [None] * count)] return list(an_args) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 1f7d8a2823f3..bcc50afc27e5 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1556,3 +1556,16 @@ ex: Example[int] = test()(reveal_type(Example())) # N: Revealed type is "__main reveal_type(test()(Example[int]())) # N: Revealed type is "__main__.Example[[builtins.int, builtins.int]]" ex = test()(Example[int]()) # E: Argument 1 has incompatible type "Example[[int]]"; expected "Example[[]]" [builtins fixtures/paramspec.pyi] + +[case testRuntimeSpecialParamspecLiteralSyntax] +import sub + +reveal_type(sub.Ex[None]()) # N: Revealed type is "sub.Ex[[None]]" +[file sub/__init__.py] +from typing_extensions import ParamSpec +from typing import Generic + +P = ParamSpec("P") + +class Ex(Generic[P]): ... +[builtins fixtures/paramspec.pyi] From d925474d0bbc8df4e1104e652e645c26a926fee0 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 15 Apr 2023 00:56:41 +0900 Subject: [PATCH 13/14] Oops, forgot to autoformat --- mypy/checkexpr.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index da608bbed0ea..0b347aba7c23 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4095,7 +4095,13 @@ def apply_type_arguments_to_callable( tp = get_proper_type(tp) if isinstance(tp, CallableType): - if len(tp.variables) == 1 and isinstance(tp.variables[0], ParamSpecType) and (len(args) != 1 or not isinstance(args[0], (Parameters, ParamSpecType, AnyType))): + if ( + len(tp.variables) == 1 + and isinstance(tp.variables[0], ParamSpecType) + and ( + len(args) != 1 or not isinstance(args[0], (Parameters, ParamSpecType, AnyType)) + ) + ): # TODO: I don't think AnyType here is valid in the general case, there's 2 cases: # 1. invalid paramspec expression (in which case we should transform it into an ellipsis) # 2. user passed it (in which case we should pass it into Parameters(...)) From b9567633ffdb145c5556d92f42c3c2d50a2889d4 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 15 Apr 2023 01:01:08 +0900 Subject: [PATCH 14/14] I really need to run the linters locally... --- mypy/checkexpr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 0b347aba7c23..c25668ba5ae4 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4099,7 +4099,10 @@ def apply_type_arguments_to_callable( len(tp.variables) == 1 and isinstance(tp.variables[0], ParamSpecType) and ( - len(args) != 1 or not isinstance(args[0], (Parameters, ParamSpecType, AnyType)) + len(args) != 1 + or not isinstance( + get_proper_type(args[0]), (Parameters, ParamSpecType, AnyType) + ) ) ): # TODO: I don't think AnyType here is valid in the general case, there's 2 cases: