From 88dffffc0af8744d2099b18aa57634d55bf0d90b Mon Sep 17 00:00:00 2001 From: layday Date: Sun, 11 Jun 2023 11:22:40 +0300 Subject: [PATCH] Make Python versions known statically Type checkers are unable to interpret `sys.version_info` aliases and generate unions between Python version branches. If either branch depends on packages which are not installed the union might be reduced to an unknown or any type. --- src/cattrs/_compat.py | 307 ++++++++++++-------------- src/cattrs/gen/typeddicts.py | 7 +- tests/_compat.py | 8 +- tests/test_baseconverter.py | 2 +- tests/test_converter.py | 2 +- tests/test_gen_dict.py | 3 +- tests/test_generics.py | 10 +- tests/test_optionals.py | 2 +- tests/test_structure_attrs.py | 2 +- tests/test_typeddicts.py | 3 +- tests/test_unions.py | 3 +- tests/test_unstructure_collections.py | 3 +- 12 files changed, 174 insertions(+), 178 deletions(-) diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index 4fa222db..e6ad59f6 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -30,25 +30,10 @@ except ImportError: ExtensionsTypedDictMeta = None -__all__ = [ - "ExtensionsTypedDict", - "is_py37", - "is_py38", - "is_py39_plus", - "is_py310_plus", - "is_py311_plus", - "is_typeddict", - "TypedDict", -] - -version_info = sys.version_info[0:3] -is_py37 = version_info[:2] == (3, 7) -is_py38 = version_info[:2] == (3, 8) -is_py39_plus = version_info[:2] >= (3, 9) -is_py310_plus = version_info[:2] >= (3, 10) -is_py311_plus = version_info[:2] >= (3, 11) - -if is_py37: +if sys.version_info >= (3, 8): + from typing import Final, Protocol, get_args, get_origin + +else: def get_args(cl): return cl.__args__ @@ -58,10 +43,7 @@ def get_origin(cl): from typing_extensions import Final, Protocol -else: - from typing import Final, Protocol, get_args, get_origin - -if is_py311_plus: +if sys.version_info >= (3, 11): ExceptionGroup = ExceptionGroup else: from exceptiongroup import ExceptionGroup as ExceptionGroup # noqa: PLC0414 @@ -157,144 +139,7 @@ def get_final_base(type) -> Optional[type]: OriginAbstractSet = AbcSet OriginMutableSet = AbcMutableSet -if is_py37 or is_py38: - Set = TypingSet - AbstractSet = TypingAbstractSet - MutableSet = TypingMutableSet - - Sequence = TypingSequence - MutableSequence = TypingMutableSequence - MutableMapping = TypingMutableMapping - Mapping = TypingMapping - FrozenSetSubscriptable = FrozenSet - TupleSubscriptable = Tuple - - from collections import Counter as ColCounter - from typing import Counter, Union, _GenericAlias - - from typing_extensions import Annotated, NotRequired, Required - from typing_extensions import get_origin as te_get_origin - - if is_py38: - from typing import TypedDict, _TypedDictMeta - else: - _TypedDictMeta = None - TypedDict = ExtensionsTypedDict - - def is_annotated(type) -> bool: - return te_get_origin(type) is Annotated - - def is_tuple(type): - return type in (Tuple, tuple) or ( - type.__class__ is _GenericAlias and issubclass(type.__origin__, Tuple) - ) - - def is_union_type(obj): - return ( - obj is Union or isinstance(obj, _GenericAlias) and obj.__origin__ is Union - ) - - def get_newtype_base(typ: Any) -> Optional[type]: - supertype = getattr(typ, "__supertype__", None) - if ( - supertype is not None - and getattr(typ, "__qualname__", "") == "NewType..new_type" - and typ.__module__ in ("typing", "typing_extensions") - ): - return supertype - return None - - def is_sequence(type: Any) -> bool: - return type in (List, list, Tuple, tuple) or ( - type.__class__ is _GenericAlias - and ( - type.__origin__ not in (Union, Tuple, tuple) - and issubclass(type.__origin__, TypingSequence) - ) - or (type.__origin__ in (Tuple, tuple) and type.__args__[1] is ...) - ) - - def is_deque(type: Any) -> bool: - return ( - type in (deque, Deque) - or (type.__class__ is _GenericAlias and issubclass(type.__origin__, deque)) - or type.__origin__ is deque - ) - - def is_mutable_set(type): - return type is set or ( - type.__class__ is _GenericAlias and issubclass(type.__origin__, MutableSet) - ) - - def is_frozenset(type): - return type is frozenset or ( - type.__class__ is _GenericAlias and issubclass(type.__origin__, FrozenSet) - ) - - def is_mapping(type): - return type in (TypingMapping, dict) or ( - type.__class__ is _GenericAlias - and issubclass(type.__origin__, TypingMapping) - ) - - bare_generic_args = { - List.__args__, - TypingSequence.__args__, - TypingMapping.__args__, - Dict.__args__, - TypingMutableSequence.__args__, - Tuple.__args__, - None, # non-parametrized containers do not have `__args__ attribute in py3.7-8 - } - - def is_bare(type): - return getattr(type, "__args__", None) in bare_generic_args - - def is_counter(type): - return ( - type in (Counter, ColCounter) - or getattr(type, "__origin__", None) is ColCounter - ) - - if is_py38: - from typing import Literal - - def is_literal(type) -> bool: - return type.__class__ is _GenericAlias and type.__origin__ is Literal - - else: - # No literals in 3.7. - def is_literal(_) -> bool: - return False - - def is_generic(obj): - return isinstance(obj, _GenericAlias) - - def copy_with(type, args): - """Replace a generic type's arguments.""" - return type.copy_with(args) - - def is_typeddict(cls) -> bool: - return ( - cls.__class__ is _TypedDictMeta - or (is_generic(cls) and (cls.__origin__.__class__ is _TypedDictMeta)) - or ( - ExtensionsTypedDictMeta is not None - and cls.__class__ is ExtensionsTypedDictMeta - or ( - is_generic(cls) - and (cls.__origin__.__class__ is ExtensionsTypedDictMeta) - ) - ) - ) - - def get_notrequired_base(type) -> "Union[Any, Literal[NOTHING]]": - if get_origin(type) in (NotRequired, Required): - return get_args(type)[0] - return NOTHING - -else: - # 3.9+ +if sys.version_info >= (3, 9): from collections import Counter from collections.abc import Mapping as AbcMapping from collections.abc import MutableMapping as AbcMutableMapping @@ -348,7 +193,7 @@ def is_tuple(type): or (getattr(type, "__origin__", None) is tuple) ) - if is_py310_plus: + if sys.version_info >= (3, 10): def is_union_type(obj): from types import UnionType @@ -364,7 +209,7 @@ def get_newtype_base(typ: Any) -> Optional[type]: return typ.__supertype__ return None - if is_py311_plus: + if sys.version_info >= (3, 11): from typing import NotRequired, Required else: from typing_extensions import NotRequired, Required @@ -500,6 +345,142 @@ def copy_with(type, args): return Annotated[args] return type.__origin__[args] +else: + Set = TypingSet + AbstractSet = TypingAbstractSet + MutableSet = TypingMutableSet + + Sequence = TypingSequence + MutableSequence = TypingMutableSequence + MutableMapping = TypingMutableMapping + Mapping = TypingMapping + FrozenSetSubscriptable = FrozenSet + TupleSubscriptable = Tuple + + from collections import Counter as ColCounter + from typing import Counter, Union, _GenericAlias + + from typing_extensions import Annotated, NotRequired, Required + from typing_extensions import get_origin as te_get_origin + + if sys.version_info >= (3, 8): + from typing import TypedDict, _TypedDictMeta + else: + _TypedDictMeta = None + TypedDict = ExtensionsTypedDict + + def is_annotated(type) -> bool: + return te_get_origin(type) is Annotated + + def is_tuple(type): + return type in (Tuple, tuple) or ( + type.__class__ is _GenericAlias and issubclass(type.__origin__, Tuple) + ) + + def is_union_type(obj): + return ( + obj is Union or isinstance(obj, _GenericAlias) and obj.__origin__ is Union + ) + + def get_newtype_base(typ: Any) -> Optional[type]: + supertype = getattr(typ, "__supertype__", None) + if ( + supertype is not None + and getattr(typ, "__qualname__", "") == "NewType..new_type" + and typ.__module__ in ("typing", "typing_extensions") + ): + return supertype + return None + + def is_sequence(type: Any) -> bool: + return type in (List, list, Tuple, tuple) or ( + type.__class__ is _GenericAlias + and ( + type.__origin__ not in (Union, Tuple, tuple) + and issubclass(type.__origin__, TypingSequence) + ) + or (type.__origin__ in (Tuple, tuple) and type.__args__[1] is ...) + ) + + def is_deque(type: Any) -> bool: + return ( + type in (deque, Deque) + or (type.__class__ is _GenericAlias and issubclass(type.__origin__, deque)) + or type.__origin__ is deque + ) + + def is_mutable_set(type): + return type is set or ( + type.__class__ is _GenericAlias and issubclass(type.__origin__, MutableSet) + ) + + def is_frozenset(type): + return type is frozenset or ( + type.__class__ is _GenericAlias and issubclass(type.__origin__, FrozenSet) + ) + + def is_mapping(type): + return type in (TypingMapping, dict) or ( + type.__class__ is _GenericAlias + and issubclass(type.__origin__, TypingMapping) + ) + + bare_generic_args = { + List.__args__, + TypingSequence.__args__, + TypingMapping.__args__, + Dict.__args__, + TypingMutableSequence.__args__, + Tuple.__args__, + None, # non-parametrized containers do not have `__args__ attribute in py3.7-8 + } + + def is_bare(type): + return getattr(type, "__args__", None) in bare_generic_args + + def is_counter(type): + return ( + type in (Counter, ColCounter) + or getattr(type, "__origin__", None) is ColCounter + ) + + if sys.version_info >= (3, 8): + from typing import Literal + + def is_literal(type) -> bool: + return type.__class__ is _GenericAlias and type.__origin__ is Literal + + else: + # No literals in 3.7. + def is_literal(_) -> bool: + return False + + def is_generic(obj): + return isinstance(obj, _GenericAlias) + + def copy_with(type, args): + """Replace a generic type's arguments.""" + return type.copy_with(args) + + def is_typeddict(cls) -> bool: + return ( + cls.__class__ is _TypedDictMeta + or (is_generic(cls) and (cls.__origin__.__class__ is _TypedDictMeta)) + or ( + ExtensionsTypedDictMeta is not None + and cls.__class__ is ExtensionsTypedDictMeta + or ( + is_generic(cls) + and (cls.__origin__.__class__ is ExtensionsTypedDictMeta) + ) + ) + ) + + def get_notrequired_base(type) -> "Union[Any, Literal[NOTHING]]": + if get_origin(type) in (NotRequired, Required): + return get_args(type)[0] + return NOTHING + def is_generic_attrs(type): return is_generic(type) and has(type.__origin__) diff --git a/src/cattrs/gen/typeddicts.py b/src/cattrs/gen/typeddicts.py index 445763a6..739eb14a 100644 --- a/src/cattrs/gen/typeddicts.py +++ b/src/cattrs/gen/typeddicts.py @@ -2,6 +2,7 @@ import linecache import re +import sys from typing import TYPE_CHECKING, Any, Callable, TypeVar from attr import NOTHING, Attribute @@ -34,8 +35,6 @@ def get_annots(cl): is_annotated, is_bare, is_generic, - is_py39_plus, - is_py311_plus, ) from .._generics import deep_copy_with from ..errors import ( @@ -533,12 +532,12 @@ def _is_extensions_typeddict(cls) -> bool: ) -if is_py311_plus: +if sys.version_info >= (3, 11): def _required_keys(cls: type) -> set[str]: return cls.__required_keys__ -elif is_py39_plus: +elif sys.version_info >= (3, 9): from typing_extensions import Annotated, NotRequired, Required, get_args def _required_keys(cls: type) -> set[str]: diff --git a/tests/_compat.py b/tests/_compat.py index 9b335c69..75ced109 100644 --- a/tests/_compat.py +++ b/tests/_compat.py @@ -1,4 +1,10 @@ -from cattrs._compat import is_py37, is_py38 +import sys + +is_py37 = sys.version_info[:2] == (3, 7) +is_py38 = sys.version_info[:2] == (3, 8) +is_py39_plus = sys.version_info >= (3, 9) +is_py310_plus = sys.version_info >= (3, 10) +is_py311_plus = sys.version_info >= (3, 11) if is_py37 or is_py38: from typing import Dict, List diff --git a/tests/test_baseconverter.py b/tests/test_baseconverter.py index 02658094..e4ac50fe 100644 --- a/tests/test_baseconverter.py +++ b/tests/test_baseconverter.py @@ -8,8 +8,8 @@ from hypothesis.strategies import just, one_of from cattrs import BaseConverter, UnstructureStrategy -from cattrs._compat import is_py310_plus +from ._compat import is_py310_plus from .typed import nested_typed_classes, simple_typed_attrs, simple_typed_classes unstructure_strats = one_of(just(s) for s in UnstructureStrategy) diff --git a/tests/test_converter.py b/tests/test_converter.py index 0266df16..1585e28c 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -19,10 +19,10 @@ from hypothesis.strategies import booleans, just, lists, one_of, sampled_from from cattrs import Converter, UnstructureStrategy -from cattrs._compat import is_py39_plus, is_py310_plus from cattrs.errors import ClassValidationError, ForbiddenExtraKeysError from cattrs.gen import make_dict_structure_fn, override +from ._compat import is_py39_plus, is_py310_plus from .typed import ( nested_typed_classes, simple_typed_attrs, diff --git a/tests/test_gen_dict.py b/tests/test_gen_dict.py index fb63a256..4d636931 100644 --- a/tests/test_gen_dict.py +++ b/tests/test_gen_dict.py @@ -8,10 +8,11 @@ from hypothesis.strategies import data, just, one_of, sampled_from from cattrs import BaseConverter, Converter -from cattrs._compat import _adapted_fields, fields, is_py39_plus +from cattrs._compat import _adapted_fields, fields from cattrs.errors import ClassValidationError, ForbiddenExtraKeysError from cattrs.gen import make_dict_structure_fn, make_dict_unstructure_fn, override +from ._compat import is_py39_plus from .typed import nested_typed_classes, simple_typed_classes, simple_typed_dataclasses from .untyped import nested_classes, simple_classes diff --git a/tests/test_generics.py b/tests/test_generics.py index c39a47b6..97e3233b 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -5,12 +5,18 @@ from attr import asdict, attrs, define from cattrs import BaseConverter, Converter -from cattrs._compat import Protocol, is_py39_plus, is_py310_plus, is_py311_plus +from cattrs._compat import Protocol from cattrs._generics import deep_copy_with from cattrs.errors import StructureHandlerNotFoundError from cattrs.gen._generics import generate_mapping -from ._compat import Dict_origin, List_origin +from ._compat import ( + Dict_origin, + List_origin, + is_py39_plus, + is_py310_plus, + is_py311_plus, +) T = TypeVar("T") T2 = TypeVar("T2") diff --git a/tests/test_optionals.py b/tests/test_optionals.py index 483b6299..510fecf0 100644 --- a/tests/test_optionals.py +++ b/tests/test_optionals.py @@ -3,7 +3,7 @@ import pytest from attrs import define -from cattrs._compat import is_py310_plus +from ._compat import is_py310_plus def test_newtype_optionals(genconverter): diff --git a/tests/test_structure_attrs.py b/tests/test_structure_attrs.py index 677a2cd3..12b3af2a 100644 --- a/tests/test_structure_attrs.py +++ b/tests/test_structure_attrs.py @@ -9,9 +9,9 @@ from hypothesis import assume, given from hypothesis.strategies import data, lists, sampled_from -from cattrs._compat import is_py37 from cattrs.converters import BaseConverter, Converter +from ._compat import is_py37 from .untyped import simple_classes diff --git a/tests/test_typeddicts.py b/tests/test_typeddicts.py index 1966bb0c..f72904a6 100644 --- a/tests/test_typeddicts.py +++ b/tests/test_typeddicts.py @@ -8,7 +8,7 @@ from pytest import raises from cattrs import Converter -from cattrs._compat import ExtensionsTypedDict, is_generic, is_py38, is_py311_plus +from cattrs._compat import ExtensionsTypedDict, is_generic from cattrs.errors import ClassValidationError, ForbiddenExtraKeysError from cattrs.gen import override from cattrs.gen._generics import generate_mapping @@ -18,6 +18,7 @@ make_dict_unstructure_fn, ) +from ._compat import is_py38, is_py311_plus from .typeddicts import ( generic_typeddicts, simple_typeddicts, diff --git a/tests/test_unions.py b/tests/test_unions.py index 34263978..3f17c31d 100644 --- a/tests/test_unions.py +++ b/tests/test_unions.py @@ -3,9 +3,10 @@ import attr import pytest -from cattrs._compat import is_py310_plus from cattrs.converters import BaseConverter, Converter +from ._compat import is_py310_plus + @pytest.mark.parametrize("cls", (BaseConverter, Converter)) def test_custom_union_toplevel_roundtrip(cls: Type[BaseConverter]): diff --git a/tests/test_unstructure_collections.py b/tests/test_unstructure_collections.py index bd5a1f61..d06287bc 100644 --- a/tests/test_unstructure_collections.py +++ b/tests/test_unstructure_collections.py @@ -14,9 +14,10 @@ from immutables import Map from cattrs import Converter -from cattrs._compat import is_py39_plus from cattrs.converters import is_mutable_set, is_sequence +from ._compat import is_py39_plus + @pytest.mark.skipif(not is_py39_plus, reason="Requires Python 3.9+") def test_collection_unstructure_override_set():