From 142a613e6142a0eb3ce23b0ca1beaa7722eea166 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 28 Oct 2023 16:11:08 +0100 Subject: [PATCH 1/4] Enable Unpack/TypeVarTuple support --- mypy/checkexpr.py | 8 ++++---- mypy/checkstrformat.py | 19 +++++++++++++++++++ mypy/main.py | 17 +++++------------ mypy/options.py | 6 +++--- mypy/semanal.py | 5 +---- mypy/test/testcheck.py | 3 --- mypy/test/testfinegrained.py | 3 +-- mypy/test/testsemanal.py | 3 +-- mypy/test/testtransform.py | 2 -- mypy/typeanal.py | 4 +--- test-data/unit/check-flags.test | 12 ------------ test-data/unit/check-tuples.test | 16 ++++++++++++++++ test-data/unit/check-typevar-tuple.test | 3 +++ test-data/unit/cmdline.test | 10 ++++++++++ 14 files changed, 64 insertions(+), 47 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index df6000050986..0207c245b1f9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -97,7 +97,7 @@ YieldExpr, YieldFromExpr, ) -from mypy.options import TYPE_VAR_TUPLE +from mypy.options import PRECISE_TUPLE_TYPES from mypy.plugin import ( FunctionContext, FunctionSigContext, @@ -3377,7 +3377,7 @@ def visit_op_expr(self, e: OpExpr) -> Type: ): return self.concat_tuples(proper_left_type, proper_right_type) elif ( - TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature + PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature and isinstance(proper_right_type, Instance) and self.chk.type_is_iterable(proper_right_type) ): @@ -3411,7 +3411,7 @@ def visit_op_expr(self, e: OpExpr) -> Type: if is_named_instance(proper_right_type, "builtins.dict"): use_reverse = USE_REVERSE_NEVER - if TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature: + if PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature: # Handle tuple[X, ...] + tuple[Y, Z] = tuple[*tuple[X, ...], Y, Z]. if ( e.op == "+" @@ -4988,7 +4988,7 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type: j += len(tt.items) else: if ( - TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature + PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature and not seen_unpack_in_items ): # Handle (x, *y, z), where y is e.g. tuple[Y, ...]. diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index eeb9e7633756..39d44e84a9c1 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -47,8 +47,11 @@ TupleType, Type, TypeOfAny, + TypeVarTupleType, TypeVarType, UnionType, + UnpackType, + find_unpack_in_list, get_proper_type, get_proper_types, ) @@ -728,6 +731,22 @@ def check_simple_str_interpolation( rep_types: list[Type] = [] if isinstance(rhs_type, TupleType): rep_types = rhs_type.items + unpack_index = find_unpack_in_list(rep_types) + if unpack_index is not None: + # TODO: we should probably warn about potentially short tuple. + # However, without special-casing for tuple(f(i) for in other_tuple) + # this causes false positive on mypy self-check in report.py. + extras = max(0, len(checkers) - len(rep_types) + 1) + unpacked = rep_types[unpack_index] + assert isinstance(unpacked, UnpackType) + unpacked = get_proper_type(unpacked.type) + if isinstance(unpacked, TypeVarTupleType): + unpacked = get_proper_type(unpacked.upper_bound) + assert ( + isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + ) + unpack_items = [unpacked.args[0]] * extras + rep_types = rep_types[:unpack_index] + unpack_items + rep_types[unpack_index + 1 :] elif isinstance(rhs_type, AnyType): return elif isinstance(rhs_type, Instance) and rhs_type.type.fullname == "builtins.tuple": diff --git a/mypy/main.py b/mypy/main.py index 43ab761072ca..1aede530c33e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -22,7 +22,7 @@ from mypy.find_sources import InvalidSourceList, create_source_list from mypy.fscache import FileSystemCache from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path -from mypy.options import INCOMPLETE_FEATURES, BuildType, Options +from mypy.options import COMPLETE_FEATURES, INCOMPLETE_FEATURES, BuildType, Options from mypy.split_namespace import SplitNamespace from mypy.version import __version__ @@ -1151,10 +1151,7 @@ def add_invertible_flag( # --debug-serialize will run tree.serialize() even if cache generation is disabled. # Useful for mypy_primer to detect serialize errors earlier. parser.add_argument("--debug-serialize", action="store_true", help=argparse.SUPPRESS) - # This one is deprecated, but we will keep it for few releases. - parser.add_argument( - "--enable-incomplete-features", action="store_true", help=argparse.SUPPRESS - ) + parser.add_argument( "--disable-bytearray-promotion", action="store_true", help=argparse.SUPPRESS ) @@ -1334,14 +1331,10 @@ def set_strict_flags() -> None: # Validate incomplete features. for feature in options.enable_incomplete_feature: - if feature not in INCOMPLETE_FEATURES: + if feature not in INCOMPLETE_FEATURES | COMPLETE_FEATURES: parser.error(f"Unknown incomplete feature: {feature}") - if options.enable_incomplete_features: - print( - "Warning: --enable-incomplete-features is deprecated, use" - " --enable-incomplete-feature=FEATURE instead" - ) - options.enable_incomplete_feature = list(INCOMPLETE_FEATURES) + if feature in COMPLETE_FEATURES: + print(f"Warning: {feature} is already enabled by default") # Compute absolute path for custom typeshed (if present). if options.custom_typeshed_dir is not None: diff --git a/mypy/options.py b/mypy/options.py index 31d5d584f897..8bb20dbd4410 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -69,11 +69,12 @@ class BuildType: } ) - {"debug_cache"} -# Features that are currently incomplete/experimental +# Features that are currently (or were recently) incomplete/experimental TYPE_VAR_TUPLE: Final = "TypeVarTuple" UNPACK: Final = "Unpack" PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" -INCOMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, PRECISE_TUPLE_TYPES)) +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES,)) +COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK)) class Options: @@ -307,7 +308,6 @@ def __init__(self) -> None: self.dump_type_stats = False self.dump_inference_stats = False self.dump_build_stats = False - self.enable_incomplete_features = False # deprecated self.enable_incomplete_feature: list[str] = [] self.timing_stats: str | None = None self.line_checking_stats: str | None = None diff --git a/mypy/semanal.py b/mypy/semanal.py index 27491ac695ae..de2cbf3f300a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -179,7 +179,7 @@ type_aliases_source_versions, typing_extensions_aliases, ) -from mypy.options import TYPE_VAR_TUPLE, Options +from mypy.options import Options from mypy.patterns import ( AsPattern, ClassPattern, @@ -4425,9 +4425,6 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: else: self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s) - if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s): - return False - name = self.extract_typevarlike_name(s, call) if name is None: return False diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 591421465a97..3ad97ced61f2 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -10,7 +10,6 @@ from mypy.build import Graph from mypy.errors import CompileError from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths -from mypy.options import TYPE_VAR_TUPLE, UNPACK from mypy.test.config import test_data_prefix, test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, module_from_path from mypy.test.helpers import ( @@ -125,8 +124,6 @@ def run_case_once( # Parse options after moving files (in case mypy.ini is being moved). options = parse_options(original_program_text, testcase, incremental_step) options.use_builtins_fixtures = True - if not testcase.name.endswith("_no_incomplete"): - options.enable_incomplete_feature += [TYPE_VAR_TUPLE, UNPACK] options.show_traceback = True # Enable some options automatically based on test file name. diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index ba0526d32558..0c69ef5bb051 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -28,7 +28,7 @@ from mypy.errors import CompileError from mypy.find_sources import create_source_list from mypy.modulefinder import BuildSource -from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options +from mypy.options import Options from mypy.server.mergecheck import check_consistency from mypy.server.update import sort_messages_preserving_file_order from mypy.test.config import test_temp_dir @@ -149,7 +149,6 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo options.use_fine_grained_cache = self.use_cache and not build_cache options.cache_fine_grained = self.use_cache options.local_partial_types = True - options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] # Treat empty bodies safely for these test cases. options.allow_empty_bodies = not testcase.name.endswith("_no_empty") if re.search("flags:.*--follow-imports", source) is None: diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index 3455f41aa20a..cdecc4739168 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -10,7 +10,7 @@ from mypy.errors import CompileError from mypy.modulefinder import BuildSource from mypy.nodes import TypeInfo -from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options +from mypy.options import Options from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import ( @@ -45,7 +45,6 @@ def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Opti options.semantic_analysis_only = True options.show_traceback = True options.python_version = PYTHON3_VERSION - options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] options.force_uppercase_builtins = True return options diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py index ba9fe8668fb4..9388dca02c7a 100644 --- a/mypy/test/testtransform.py +++ b/mypy/test/testtransform.py @@ -5,7 +5,6 @@ from mypy import build from mypy.errors import CompileError from mypy.modulefinder import BuildSource -from mypy.options import TYPE_VAR_TUPLE, UNPACK from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages, parse_options @@ -38,7 +37,6 @@ def test_transform(testcase: DataDrivenTestCase) -> None: options = parse_options(src, testcase, 1) options.use_builtins_fixtures = True options.semantic_analysis_only = True - options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] options.show_traceback = True options.force_uppercase_builtins = True result = build.build( diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 03579404aac9..d238a452e7a9 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -35,7 +35,7 @@ check_arg_names, get_nongen_builtins, ) -from mypy.options import UNPACK, Options +from mypy.options import Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs from mypy.tvar_scope import TypeVarLikeScope @@ -664,8 +664,6 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ # In most contexts, TypeGuard[...] acts as an alias for bool (ignoring its args) return self.named_type("builtins.bool") elif fullname in ("typing.Unpack", "typing_extensions.Unpack"): - if not self.api.incomplete_feature_enabled(UNPACK, t): - return AnyType(TypeOfAny.from_error) if len(t.args) != 1: self.fail("Unpack[...] requires exactly one type argument", t) return AnyType(TypeOfAny.from_error) diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index 546d02a07ad0..04adaca317c1 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -2190,18 +2190,6 @@ x: int = "" # E: Incompatible types in assignment (expression has type "str", v # flags: --hide-error-codes x: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[case testTypeVarTupleDisabled_no_incomplete] -from typing_extensions import TypeVarTuple -Ts = TypeVarTuple("Ts") # E: "TypeVarTuple" support is experimental, use --enable-incomplete-feature=TypeVarTuple to enable -[builtins fixtures/tuple.pyi] - -[case testTypeVarTupleEnabled_no_incomplete] -# flags: --enable-incomplete-feature=TypeVarTuple -from typing_extensions import TypeVarTuple -Ts = TypeVarTuple("Ts") # OK -[builtins fixtures/tuple.pyi] - - [case testDisableBytearrayPromotion] # flags: --disable-bytearray-promotion def f(x: bytes) -> None: ... diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 7070ead43746..4f468b59fc3f 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1100,12 +1100,28 @@ reveal_type(b) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtin [case testTupleWithStarExpr2] a = [1] b = (0, *a) +reveal_type(b) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr2Precise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +a = [1] +b = (0, *a) reveal_type(b) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]" [builtins fixtures/tuple.pyi] [case testTupleWithStarExpr3] a = [''] b = (0, *a) +reveal_type(b) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +c = (*a, '') +reveal_type(c) # N: Revealed type is "builtins.tuple[builtins.str, ...]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr3Precise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +a = [''] +b = (0, *a) reveal_type(b) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.str, ...]]]" c = (*a, '') reveal_type(c) # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.str, ...]], builtins.str]" diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 7b8a22313b36..a51b535a873c 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1653,6 +1653,7 @@ def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: [builtins fixtures/tuple.pyi] [case testPackingVariadicTuplesHomogeneous] +# flags: --enable-incomplete-feature=PreciseTupleTypes from typing import Tuple from typing_extensions import Unpack @@ -1689,6 +1690,7 @@ def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: [builtins fixtures/isinstancelist.pyi] [case testVariadicTupleInTupleContext] +# flags: --enable-incomplete-feature=PreciseTupleTypes from typing import Tuple, Optional from typing_extensions import TypeVarTuple, Unpack @@ -1701,6 +1703,7 @@ vt2 = 1, *test(), 2 # E: Need type annotation for "vt2" [builtins fixtures/tuple.pyi] [case testVariadicTupleConcatenation] +# flags: --enable-incomplete-feature=PreciseTupleTypes from typing import Tuple from typing_extensions import TypeVarTuple, Unpack diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 91242eb62fcf..7a46c91220bf 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1585,3 +1585,13 @@ disable_error_code = always_true = MY_VAR, [out] + +[case testTypeVarTupleUnpackEnabled] +# cmd: mypy --enable-incomplete-feature=TypeVarTuple --enable-incomplete-feature=Unpack a.py +[file a.py] +from typing_extensions import TypeVarTuple +Ts = TypeVarTuple("Ts") +[out] +Warning: TypeVarTuple is already enabled by default +Warning: Unpack is already enabled by default +== Return code: 0 From af0e687df8890b6a9956242b6149257401b7d2c7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 28 Oct 2023 16:50:07 +0100 Subject: [PATCH 2/4] Add docs about the new incomplete feature --- docs/source/command_line.rst | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 4e954c7c2ccb..f3ea69b785d9 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -991,6 +991,58 @@ format into the specified directory. library or specify mypy installation with the setuptools extra ``mypy[reports]``. + +Enabling incomplete/experimental features +***************************************** + +.. option:: --enable-incomplete-feature FEATURE + + Some features may require several mypy releases to implement, for example + due to their complexity, potential for backwards incompatibility, or + ambiguous semantics that would benefit from feedback from the community. + You can enable such features for early preview using this flag. Note that + it is not guaranteed that all features will be ultimately enabled by + default, in *rare cases* we may decide to not go ahead with certain + features. + +List of currently incomplete/experimental features: + +* ``PreciseTupleTypes``: this feature will infer more precise tuple types in + various scenarios. Before variadic types were added to Python type system + by :pep:`646`, it was impossible to express a type like "a tuple with + at least two integers". The best type available was ``tuple[int, ...]``, + therefore, mypy applied very lenient checking for variable-length tuples. + Now this type can be expressed as ``tuple[int, int, *tuple[int, ...]]``. + For such more precise types (when explicitly *defined* by a user) mypy, + for example, warns about unsafe index access, and generally handles them + in a type-safe manner. However, to avoid problems in existing code, mypy + does not *infer* these precise types when it technically can. Here are + notable examples where ``PreciseTupleTypes`` infers more precise types: + + .. code-block:: python + + numbers: tuple[int, ...] + + more_numbers = (1, *numbers, 1) + reveal_type(more_numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, *tuple[int, ...], int] + + other_numbers = (1, 1) + numbers + reveal_type(other_numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, int, *tuple[int, ...]] + + if len(numbers) > 2: + reveal_type(numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, int, int, *tuple[int, ...]] + else: + reveal_type(numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int] + + Miscellaneous ************* From 760e739924681ff220afbd1997ab6b89e2f34d13 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 28 Oct 2023 17:07:09 +0100 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Jelle Zijlstra --- docs/source/command_line.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index f3ea69b785d9..58b20f6ee785 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1002,16 +1002,16 @@ Enabling incomplete/experimental features ambiguous semantics that would benefit from feedback from the community. You can enable such features for early preview using this flag. Note that it is not guaranteed that all features will be ultimately enabled by - default, in *rare cases* we may decide to not go ahead with certain + default. In *rare cases* we may decide to not go ahead with certain features. List of currently incomplete/experimental features: * ``PreciseTupleTypes``: this feature will infer more precise tuple types in - various scenarios. Before variadic types were added to Python type system + various scenarios. Before variadic types were added to the Python type system by :pep:`646`, it was impossible to express a type like "a tuple with - at least two integers". The best type available was ``tuple[int, ...]``, - therefore, mypy applied very lenient checking for variable-length tuples. + at least two integers". The best type available was ``tuple[int, ...]``. + Therefore, mypy applied very lenient checking for variable-length tuples. Now this type can be expressed as ``tuple[int, int, *tuple[int, ...]]``. For such more precise types (when explicitly *defined* by a user) mypy, for example, warns about unsafe index access, and generally handles them From 305dd9100bc76e7f898b0ce09fa8930e8ecf9ea3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 28 Oct 2023 17:39:51 +0100 Subject: [PATCH 4/4] Delete outdated test --- test-data/unit/cmdline.test | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 7a46c91220bf..f286f4781ed5 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1421,14 +1421,6 @@ b \d+ b\.c \d+ .* -[case testCmdlineEnableIncompleteFeatures] -# cmd: mypy --enable-incomplete-features a.py -[file a.py] -pass -[out] -Warning: --enable-incomplete-features is deprecated, use --enable-incomplete-feature=FEATURE instead -== Return code: 0 - [case testShadowTypingModuleEarlyLoad] # cmd: mypy dir [file dir/__init__.py]