From b5f8f01df72611c13dbd751c518fe50618adadc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ila=C3=AF=20Deutel?= Date: Tue, 15 Oct 2019 21:19:51 -0700 Subject: [PATCH 1/5] Removing open plugin, not needed with overloaded signatures on typeshed --- mypy/plugins/default.py | 56 --------------------------------- mypy/typeshed | 2 +- test-data/unit/python2eval.test | 1 - test-data/unit/pythoneval.test | 12 ++++--- 4 files changed, 9 insertions(+), 62 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index ca9d3baad3bb..a2689f86754e 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -25,8 +25,6 @@ def get_function_hook(self, fullname: str if fullname == 'contextlib.contextmanager': return contextmanager_callback - elif fullname == 'builtins.open' and self.python_version[0] == 3: - return open_callback elif fullname == 'ctypes.Array': return ctypes.array_constructor_callback return None @@ -67,8 +65,6 @@ def get_method_hook(self, fullname: str return ctypes.array_getitem_callback elif fullname == 'ctypes.Array.__iter__': return ctypes.array_iter_callback - elif fullname == 'pathlib.Path.open': - return path_open_callback return None def get_attribute_hook(self, fullname: str @@ -103,58 +99,6 @@ def get_class_decorator_hook(self, fullname: str return None -def open_callback(ctx: FunctionContext) -> Type: - """Infer a better return type for 'open'.""" - return _analyze_open_signature( - arg_types=ctx.arg_types, - args=ctx.args, - mode_arg_index=1, - default_return_type=ctx.default_return_type, - api=ctx.api, - ) - - -def path_open_callback(ctx: MethodContext) -> Type: - """Infer a better return type for 'pathlib.Path.open'.""" - return _analyze_open_signature( - arg_types=ctx.arg_types, - args=ctx.args, - mode_arg_index=0, - default_return_type=ctx.default_return_type, - api=ctx.api, - ) - - -def _analyze_open_signature(arg_types: List[List[Type]], - args: List[List[Expression]], - mode_arg_index: int, - default_return_type: Type, - api: CheckerPluginInterface, - ) -> Type: - """A helper for analyzing any function that has approximately - the same signature as the builtin 'open(...)' function. - - Currently, the only thing the caller can customize is the index - of the 'mode' argument. If the mode argument is omitted or is a - string literal, we refine the return type to either 'TextIO' or - 'BinaryIO' as appropriate. - """ - mode = None - if not arg_types or len(arg_types[mode_arg_index]) != 1: - mode = 'r' - else: - mode_expr = args[mode_arg_index][0] - if isinstance(mode_expr, StrExpr): - mode = mode_expr.value - if mode is not None: - assert isinstance(default_return_type, Instance) # type: ignore - if 'b' in mode: - return api.named_generic_type('typing.BinaryIO', []) - else: - return api.named_generic_type('typing.TextIO', []) - return default_return_type - - def contextmanager_callback(ctx: FunctionContext) -> Type: """Infer a better return type for 'contextlib.contextmanager'.""" # Be defensive, just in case. diff --git a/mypy/typeshed b/mypy/typeshed index 7c6104ddfe00..1eb282ed45f2 160000 --- a/mypy/typeshed +++ b/mypy/typeshed @@ -1 +1 @@ -Subproject commit 7c6104ddfe000b76cb52b1bc01046138c233aeec +Subproject commit 1eb282ed45f279c5c005f3bfb4f2eaa53c0942fc diff --git a/test-data/unit/python2eval.test b/test-data/unit/python2eval.test index 2267cadb1a08..9c983c3f86e3 100644 --- a/test-data/unit/python2eval.test +++ b/test-data/unit/python2eval.test @@ -189,7 +189,6 @@ f.write(u'foo') f.write('bar') f.close() [out] -_program.py:3: error: Argument 1 to "write" of "IO" has incompatible type "unicode"; expected "str" [case testPrintFunctionWithFileArg_python2] from __future__ import print_function diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index f3a88ca47dcc..02472e0ce94d 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -293,8 +293,12 @@ reveal_type(open(file='x', mode='rb')) mode = 'rb' reveal_type(open(mode=mode, file='r')) [out] -_testOpenReturnTypeInferenceSpecialCases.py:1: error: Too few arguments for "open" -_testOpenReturnTypeInferenceSpecialCases.py:1: note: Revealed type is 'typing.TextIO' +_testOpenReturnTypeInferenceSpecialCases.py:1: error: All overload variants of "open" require at least one argument +_testOpenReturnTypeInferenceSpecialCases.py:1: note: Possible overload variants: +_testOpenReturnTypeInferenceSpecialCases.py:1: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: = ..., buffering: int = ..., encoding: Optional[str] = ..., errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> TextIO +_testOpenReturnTypeInferenceSpecialCases.py:1: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: , buffering: int = ..., encoding: None = ..., errors: None = ..., newline: None = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> BinaryIO +_testOpenReturnTypeInferenceSpecialCases.py:1: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: str, buffering: int = ..., encoding: Optional[str] = ..., errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> IO[Any] +_testOpenReturnTypeInferenceSpecialCases.py:1: note: Revealed type is 'Any' _testOpenReturnTypeInferenceSpecialCases.py:2: note: Revealed type is 'typing.BinaryIO' _testOpenReturnTypeInferenceSpecialCases.py:3: note: Revealed type is 'typing.BinaryIO' _testOpenReturnTypeInferenceSpecialCases.py:5: note: Revealed type is 'typing.IO[Any]' @@ -321,8 +325,8 @@ reveal_type(p.open(errors='replace', mode='rb')) mode = 'rb' reveal_type(p.open(mode=mode, errors='replace')) [out] -_program.py:3: note: Revealed type is 'typing.BinaryIO' -_program.py:4: note: Revealed type is 'typing.BinaryIO' +_program.py:3: note: Revealed type is 'typing.IO[Any]' +_program.py:4: note: Revealed type is 'typing.IO[Any]' _program.py:6: note: Revealed type is 'typing.IO[Any]' [case testGenericPatterns] From 403e0efa540ac6a555664e7b2356a7f3ef096888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ila=C3=AF=20Deutel?= Date: Thu, 24 Oct 2019 23:35:42 -0700 Subject: [PATCH 2/5] Remove unused imports in mypy/plugins/default.py --- mypy/plugins/default.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index a2689f86754e..16ba3defe1f1 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -2,14 +2,13 @@ from typing import Callable, Optional, List from mypy import message_registry -from mypy.nodes import Expression, StrExpr, IntExpr, DictExpr, UnaryExpr +from mypy.nodes import StrExpr, IntExpr, DictExpr, UnaryExpr from mypy.plugin import ( Plugin, FunctionContext, MethodContext, MethodSigContext, AttributeContext, ClassDefContext, - CheckerPluginInterface, ) from mypy.plugins.common import try_getting_str_literals from mypy.types import ( - Type, Instance, AnyType, TypeOfAny, CallableType, NoneType, TypedDictType, + Type, AnyType, TypeOfAny, CallableType, NoneType, TypedDictType, TypeVarType, TPDICT_FB_NAMES, get_proper_type ) from mypy.subtypes import is_subtype From 31d37193b792c87201192fe810164ab20a10610c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ila=C3=AF=20Deutel?= Date: Sun, 10 Nov 2019 23:29:46 -0800 Subject: [PATCH 3/5] Add plugin to verify some args for open() are None in binary mode This reverts commit b5f8f01df72611c13dbd751c518fe50618adadc9. --- mypy/plugins/default.py | 71 +++++++++++++++++++++++++++++++++- test-data/unit/pythoneval.test | 53 +++++++++++++++++-------- 2 files changed, 106 insertions(+), 18 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 16ba3defe1f1..1ac207b1c504 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -1,10 +1,11 @@ from functools import partial -from typing import Callable, Optional, List +from typing import Callable, Optional, List, Tuple from mypy import message_registry -from mypy.nodes import StrExpr, IntExpr, DictExpr, UnaryExpr +from mypy.nodes import Context, Expression, StrExpr, IntExpr, DictExpr, UnaryExpr from mypy.plugin import ( Plugin, FunctionContext, MethodContext, MethodSigContext, AttributeContext, ClassDefContext, + CheckerPluginInterface, ) from mypy.plugins.common import try_getting_str_literals from mypy.types import ( @@ -24,6 +25,8 @@ def get_function_hook(self, fullname: str if fullname == 'contextlib.contextmanager': return contextmanager_callback + elif fullname == 'builtins.open' and self.python_version[0] == 3: + return open_callback elif fullname == 'ctypes.Array': return ctypes.array_constructor_callback return None @@ -64,6 +67,8 @@ def get_method_hook(self, fullname: str return ctypes.array_getitem_callback elif fullname == 'ctypes.Array.__iter__': return ctypes.array_iter_callback + elif fullname == 'pathlib.Path.open': + return path_open_callback return None def get_attribute_hook(self, fullname: str @@ -98,6 +103,68 @@ def get_class_decorator_hook(self, fullname: str return None +def open_callback(ctx: FunctionContext) -> Type: + """Verify argument types for 'open'.""" + _verify_open_signature( + arg_types=ctx.arg_types, + args=ctx.args, + arg_names=ctx.callee_arg_names, + mode_arg_index=1, + text_only_arg_indices=(3, 4, 5), + api=ctx.api, + context=ctx.context, + ) + return ctx.default_return_type + + +def path_open_callback(ctx: MethodContext) -> Type: + """Verify argument types for 'pathlib.Path.open'.""" + _verify_open_signature( + arg_types=ctx.arg_types, + args=ctx.args, + arg_names=ctx.callee_arg_names, + mode_arg_index=0, + text_only_arg_indices=(2, 3, 4), + api=ctx.api, + context=ctx.context, + ) + return ctx.default_return_type + + +def _verify_open_signature( + arg_types: List[List[Type]], + args: List[List[Expression]], + arg_names: List[Optional[str]], + mode_arg_index: int, + text_only_arg_indices: Tuple[int, ...], + api: CheckerPluginInterface, + context: Context, +) -> None: + """A helper for verifying any function that has approximately the same + signature as the builtin 'open(...)' function. + + If mode is detected to be binary, verify that text-only arguments + (encoding, errors, newline) are None. + """ + if not arg_types or len(arg_types[mode_arg_index]) != 1: + return None + + mode_str = try_getting_str_literals( + args[mode_arg_index][0], arg_types[mode_arg_index][0] + ) + if mode_str is None or len(mode_str) != 1 or 'b' not in mode_str[0]: + # mode cannot be found, or is not binary + return None + + # Verify that text-only arguments are None if mode is binary + for arg_index in text_only_arg_indices: + if (len(arg_types[arg_index]) == 1 + and not is_subtype(arg_types[arg_index][0], NoneType())): + api.fail("Binary mode doesn't take an argument for {}" + .format(arg_names[arg_index]), + context) + + def contextmanager_callback(ctx: FunctionContext) -> Type: """Infer a better return type for 'contextlib.contextmanager'.""" # Be defensive, just in case. diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 02472e0ce94d..c4f75c8d38a4 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -287,21 +287,32 @@ _program.py:3: note: Revealed type is 'typing.BinaryIO' _program.py:5: note: Revealed type is 'typing.IO[Any]' [case testOpenReturnTypeInferenceSpecialCases] +from typing_extensions import Literal reveal_type(open()) reveal_type(open(mode='rb', file='x')) reveal_type(open(file='x', mode='rb')) mode = 'rb' -reveal_type(open(mode=mode, file='r')) -[out] -_testOpenReturnTypeInferenceSpecialCases.py:1: error: All overload variants of "open" require at least one argument -_testOpenReturnTypeInferenceSpecialCases.py:1: note: Possible overload variants: -_testOpenReturnTypeInferenceSpecialCases.py:1: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: = ..., buffering: int = ..., encoding: Optional[str] = ..., errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> TextIO -_testOpenReturnTypeInferenceSpecialCases.py:1: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: , buffering: int = ..., encoding: None = ..., errors: None = ..., newline: None = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> BinaryIO -_testOpenReturnTypeInferenceSpecialCases.py:1: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: str, buffering: int = ..., encoding: Optional[str] = ..., errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> IO[Any] -_testOpenReturnTypeInferenceSpecialCases.py:1: note: Revealed type is 'Any' -_testOpenReturnTypeInferenceSpecialCases.py:2: note: Revealed type is 'typing.BinaryIO' +reveal_type(open(mode=mode, file='r', errors='replace')) +open('x', mode='rb', errors='replace') +open('x', errors='replace', mode='rb') +errors: Literal['strict', 'ignore'] = 'ignore' +open('x', 'rb', -1, 'UTF-8', errors, '\n') +[out] +_testOpenReturnTypeInferenceSpecialCases.py:2: error: All overload variants of "open" require at least one argument +_testOpenReturnTypeInferenceSpecialCases.py:2: note: Possible overload variants: +_testOpenReturnTypeInferenceSpecialCases.py:2: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: = ..., buffering: int = ..., encoding: Optional[str] = ..., errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> TextIO +_testOpenReturnTypeInferenceSpecialCases.py:2: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: , buffering: int = ..., encoding: None = ..., errors: None = ..., newline: None = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> BinaryIO +_testOpenReturnTypeInferenceSpecialCases.py:2: note: def open(file: Union[str, bytes, int, _PathLike[Any]], mode: str, buffering: int = ..., encoding: Optional[str] = ..., errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ..., opener: Optional[Callable[[str, int], int]] = ...) -> IO[Any] +_testOpenReturnTypeInferenceSpecialCases.py:2: note: Revealed type is 'Any' _testOpenReturnTypeInferenceSpecialCases.py:3: note: Revealed type is 'typing.BinaryIO' -_testOpenReturnTypeInferenceSpecialCases.py:5: note: Revealed type is 'typing.IO[Any]' +_testOpenReturnTypeInferenceSpecialCases.py:4: note: Revealed type is 'typing.BinaryIO' +_testOpenReturnTypeInferenceSpecialCases.py:6: note: Revealed type is 'typing.IO[Any]' +_testOpenReturnTypeInferenceSpecialCases.py:7: error: Binary mode doesn't take an argument for errors +_testOpenReturnTypeInferenceSpecialCases.py:8: error: Binary mode doesn't take an argument for errors +_testOpenReturnTypeInferenceSpecialCases.py:10: error: Binary mode doesn't take an argument for encoding +_testOpenReturnTypeInferenceSpecialCases.py:10: error: Binary mode doesn't take an argument for errors +_testOpenReturnTypeInferenceSpecialCases.py:10: error: Binary mode doesn't take an argument for newline + [case testPathOpenReturnTypeInference] from pathlib import Path @@ -319,15 +330,25 @@ _program.py:7: note: Revealed type is 'typing.IO[Any]' [case testPathOpenReturnTypeInferenceSpecialCases] from pathlib import Path +from typing import Optional +from typing_extensions import Literal p = Path("x") -reveal_type(p.open(mode='rb', errors='replace')) -reveal_type(p.open(errors='replace', mode='rb')) +p.open(mode='rb', errors='replace') +p.open(errors='replace', mode='rb') mode = 'rb' reveal_type(p.open(mode=mode, errors='replace')) -[out] -_program.py:3: note: Revealed type is 'typing.IO[Any]' -_program.py:4: note: Revealed type is 'typing.IO[Any]' -_program.py:6: note: Revealed type is 'typing.IO[Any]' +mode2: Literal['rb'] = 'rb' +p.open(mode=mode2, errors='replace') +errors: Optional[Literal['strict', 'ignore']] = None +p.open('rb', -1, 'UTF-8', errors, '\n') +[out] +_program.py:5: error: Binary mode doesn't take an argument for errors +_program.py:6: error: Binary mode doesn't take an argument for errors +_program.py:8: note: Revealed type is 'typing.IO[Any]' +_program.py:10: error: Binary mode doesn't take an argument for errors +_program.py:12: error: Binary mode doesn't take an argument for encoding +_program.py:12: error: Binary mode doesn't take an argument for errors +_program.py:12: error: Binary mode doesn't take an argument for newline [case testGenericPatterns] from typing import Pattern From e568e21940bda3562df66351e64bb2695859c92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ila=C3=AF=20Deutel?= Date: Tue, 12 Nov 2019 00:25:47 -0800 Subject: [PATCH 4/5] Update typeshed; includes fix for python/typeshed#3446 --- mypy/typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/typeshed b/mypy/typeshed index e6a43215cea5..3a631c048f23 160000 --- a/mypy/typeshed +++ b/mypy/typeshed @@ -1 +1 @@ -Subproject commit e6a43215cea57c5b4448183c2b4d20f40cbcc222 +Subproject commit 3a631c048f23cf2d92c146595de5d21ba0a6592d From feb0a871d1457e76f852cceac50cc1c03e34f8e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ila=C3=AF=20Deutel?= Date: Wed, 15 Jan 2020 09:37:41 -0800 Subject: [PATCH 5/5] Sync typeshed, fix testOpenReturnType_python2 --- mypy/plugins/default.py | 4 ++-- mypy/typeshed | 2 +- test-data/unit/python2eval.test | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 5c78d2946902..6dbd48253e8a 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -150,14 +150,14 @@ def _verify_open_signature( (encoding, errors, newline) are None. """ if not arg_types or len(arg_types[mode_arg_index]) != 1: - return None + return mode_str = try_getting_str_literals( args[mode_arg_index][0], arg_types[mode_arg_index][0] ) if mode_str is None or len(mode_str) != 1 or 'b' not in mode_str[0]: # mode cannot be found, or is not binary - return None + return # Verify that text-only arguments are None if mode is binary for arg_index in text_only_arg_indices: diff --git a/mypy/typeshed b/mypy/typeshed index f924ef2394e8..7455811ca5d6 160000 --- a/mypy/typeshed +++ b/mypy/typeshed @@ -1 +1 @@ -Subproject commit f924ef2394e81e1673608b0711f1fcc11ff4f9c0 +Subproject commit 7455811ca5d644a01cdfbf250732a1c77d595ecd diff --git a/test-data/unit/python2eval.test b/test-data/unit/python2eval.test index 1bd3aa07f28c..93fe668a8b81 100644 --- a/test-data/unit/python2eval.test +++ b/test-data/unit/python2eval.test @@ -189,6 +189,7 @@ f.write(u'foo') f.write('bar') f.close() [out] +_program.py:3: error: Argument 1 to "write" of "IO" has incompatible type "unicode"; expected "str" [case testPrintFunctionWithFileArg_python2] from __future__ import print_function