From 0facfb758afd5d74e026b4d99fb6d9c80d8f0fd3 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 16 Jul 2023 11:31:47 -0400 Subject: [PATCH 01/14] Add compatibility with astroid inference cache changes --- pylint/checkers/typecheck.py | 13 +++++-------- pylint/checkers/utils.py | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pylint/checkers/typecheck.py b/pylint/checkers/typecheck.py index f95bf4c2ed..2cef6fc75b 100644 --- a/pylint/checkers/typecheck.py +++ b/pylint/checkers/typecheck.py @@ -1899,16 +1899,13 @@ def visit_with(self, node: nodes.With) -> None: # Retrieve node from all previously visited nodes in the # inference history - context_path_names: Iterator[Any] = filter( - None, _unflatten(context.path) - ) - inferred_paths = _flatten_container( - safe_infer(path) for path in context_path_names - ) - for inferred_path in inferred_paths: + for inferred_path, _ in context.path: if not inferred_path: continue - scope = inferred_path.scope() + if isinstance(inferred_path, nodes.Call): + scope = safe_infer(inferred_path.func) + else: + scope = inferred_path.scope() if not isinstance(scope, nodes.FunctionDef): continue if decorated_with( diff --git a/pylint/checkers/utils.py b/pylint/checkers/utils.py index edb0aa6c3b..ac33e93ec0 100644 --- a/pylint/checkers/utils.py +++ b/pylint/checkers/utils.py @@ -1471,7 +1471,7 @@ def node_type(node: nodes.NodeNG) -> SuccessfulInferenceResult | None: # don't handle it for now types: set[SuccessfulInferenceResult] = set() try: - for var_type in node.infer(): + for var_type in node.infer(context=InferenceContext()): if isinstance(var_type, util.UninferableBase) or is_none(var_type): continue types.add(var_type) From 077e37df2bf1fe981fb8c7199ea3c7fb177dc339 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 23 Jul 2023 09:40:29 -0700 Subject: [PATCH 02/14] Pin to astroid commit with inference cache changes --- doc/requirements.txt | 1 + pyproject.toml | 2 +- requirements_test_min.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 4d20cfbb48..92d284d299 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,3 +3,4 @@ sphinx-reredirects<1 towncrier~=23.6 furo==2023.5.20 -e . +astroid @ git+https://github.com/pylint-dev/pylint.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2 diff --git a/pyproject.toml b/pyproject.toml index f81a7933ed..63fa88acc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ # Also upgrade requirements_test_min.txt. # Pinned to dev of second minor update to allow editable installs and fix primer issues, # see https://github.com/pylint-dev/astroid/issues/1341 - "astroid>=3.0.0a8,<=3.1.0-dev0", + "astroid @ git+https://github.com/pylint-dev/pylint.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2", "isort>=4.2.5,<6", "mccabe>=0.6,<0.8", "tomli>=1.1.0;python_version<'3.11'", diff --git a/requirements_test_min.txt b/requirements_test_min.txt index 7df78810f2..d83c9a0ca6 100644 --- a/requirements_test_min.txt +++ b/requirements_test_min.txt @@ -1,6 +1,6 @@ -e .[testutils,spelling] # astroid dependency is also defined in pyproject.toml -astroid==3.0.0a8 # Pinned to a specific version for tests +astroid @ git+https://github.com/pylint-dev/pylint.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2 typing-extensions~=4.7 py~=1.11.0 pytest~=7.4 From f99875f11dd8a32a191321604e87689bfeba70e1 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 23 Jul 2023 09:50:27 -0700 Subject: [PATCH 03/14] fixup! Pin --- doc/requirements.txt | 2 +- pyproject.toml | 2 +- requirements_test_min.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 92d284d299..289105e5c1 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,4 +3,4 @@ sphinx-reredirects<1 towncrier~=23.6 furo==2023.5.20 -e . -astroid @ git+https://github.com/pylint-dev/pylint.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2 +astroid @ git+https://github.com/pylint-dev/astroid.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2 diff --git a/pyproject.toml b/pyproject.toml index 63fa88acc8..dae9cfed5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ # Also upgrade requirements_test_min.txt. # Pinned to dev of second minor update to allow editable installs and fix primer issues, # see https://github.com/pylint-dev/astroid/issues/1341 - "astroid @ git+https://github.com/pylint-dev/pylint.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2", + "astroid @ git+https://github.com/pylint-dev/astroid.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2", "isort>=4.2.5,<6", "mccabe>=0.6,<0.8", "tomli>=1.1.0;python_version<'3.11'", diff --git a/requirements_test_min.txt b/requirements_test_min.txt index d83c9a0ca6..b2b928f98d 100644 --- a/requirements_test_min.txt +++ b/requirements_test_min.txt @@ -1,6 +1,6 @@ -e .[testutils,spelling] # astroid dependency is also defined in pyproject.toml -astroid @ git+https://github.com/pylint-dev/pylint.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2 +astroid @ git+https://github.com/pylint-dev/astroid.git@cf8763a2b8e897ec7c8389906f3cb13714300cd2 typing-extensions~=4.7 py~=1.11.0 pytest~=7.4 From 80698fc51886b4178e0e454701084e0f7d731f8a Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Tue, 1 Aug 2023 09:25:08 -0400 Subject: [PATCH 04/14] Add news --- doc/whatsnew/fragments/8872.performance | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/whatsnew/fragments/8872.performance diff --git a/doc/whatsnew/fragments/8872.performance b/doc/whatsnew/fragments/8872.performance new file mode 100644 index 0000000000..69f4a4f1d8 --- /dev/null +++ b/doc/whatsnew/fragments/8872.performance @@ -0,0 +1,4 @@ +``pylint`` runs (at least) ~5% faster after improvements to ``astroid`` +that make better use of the inference cache. + +Refs pylint-dev/astroid#529 From 81be03b6ed0baabfbb739244872bb8caa7f4ddd5 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 5 Aug 2023 13:15:13 -0400 Subject: [PATCH 05/14] Address primer result for properties --- pylint/checkers/design_analysis.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py index bea1ef53cc..83b2c4383b 100644 --- a/pylint/checkers/design_analysis.py +++ b/pylint/checkers/design_analysis.py @@ -440,11 +440,18 @@ def visit_classdef(self, node: nodes.ClassDef) -> None: args=(nb_parents, self.linter.config.max_parents), ) - if len(node.instance_attrs) > self.linter.config.max_attributes: + # Something at inference time is modifying instance_attrs to add + # properties from parent classes. Given how much we cache inference + # results, mutating instance_attrs can become a real mess. Filter + # them out here until the root cause is solved. + # https://github.com/pylint-dev/astroid/issues/2273 + filtered_attrs = [a for a in node.instance_attrs if a in node.locals] + + if len(filtered_attrs) > self.linter.config.max_attributes: self.add_message( "too-many-instance-attributes", node=node, - args=(len(node.instance_attrs), self.linter.config.max_attributes), + args=(len(filtered_attrs), self.linter.config.max_attributes), ) @only_required_for_messages("too-few-public-methods", "too-many-public-methods") From 84aebc9ef49e02be2209fdb56f364f1d6938cdc0 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 5 Aug 2023 13:22:23 -0400 Subject: [PATCH 06/14] Exclude file from news fragment check --- doc/whatsnew/fragments/{8872.performance => _.8872.performance} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/whatsnew/fragments/{8872.performance => _.8872.performance} (100%) diff --git a/doc/whatsnew/fragments/8872.performance b/doc/whatsnew/fragments/_.8872.performance similarity index 100% rename from doc/whatsnew/fragments/8872.performance rename to doc/whatsnew/fragments/_.8872.performance From 81a4854e2ed9b41099bafa7dc6571a074d805bb9 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 5 Aug 2023 13:27:40 -0400 Subject: [PATCH 07/14] Just cheat and use the astroid issue number --- doc/whatsnew/fragments/{_.8872.performance => 529.performance} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/whatsnew/fragments/{_.8872.performance => 529.performance} (100%) diff --git a/doc/whatsnew/fragments/_.8872.performance b/doc/whatsnew/fragments/529.performance similarity index 100% rename from doc/whatsnew/fragments/_.8872.performance rename to doc/whatsnew/fragments/529.performance From 9e250bb975636c6c83b95ee1882710bad3bf29e3 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 5 Aug 2023 13:49:34 -0400 Subject: [PATCH 08/14] Fix implementation --- pylint/checkers/design_analysis.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py index 83b2c4383b..df45a83d6b 100644 --- a/pylint/checkers/design_analysis.py +++ b/pylint/checkers/design_analysis.py @@ -15,7 +15,11 @@ from astroid import nodes from pylint.checkers import BaseChecker -from pylint.checkers.utils import is_enum, only_required_for_messages +from pylint.checkers.utils import ( + is_enum, + get_node_first_ancestor_of_type, + only_required_for_messages, +) from pylint.typing import MessageDefinitionTuple if TYPE_CHECKING: @@ -445,8 +449,12 @@ def visit_classdef(self, node: nodes.ClassDef) -> None: # results, mutating instance_attrs can become a real mess. Filter # them out here until the root cause is solved. # https://github.com/pylint-dev/astroid/issues/2273 - filtered_attrs = [a for a in node.instance_attrs if a in node.locals] - + this_module = get_node_first_ancestor_of_type(node, nodes.Module) + filtered_attrs = [ + k + for (k, v) in node.instance_attrs.items() + if get_node_first_ancestor_of_type(v[0], nodes.Module) is this_module + ] if len(filtered_attrs) > self.linter.config.max_attributes: self.add_message( "too-many-instance-attributes", From b3e563ca004a8c151e5f42f2134a7b7aaa7a987c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 5 Aug 2023 17:51:20 +0000 Subject: [PATCH 09/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pylint/checkers/design_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py index df45a83d6b..4748828a09 100644 --- a/pylint/checkers/design_analysis.py +++ b/pylint/checkers/design_analysis.py @@ -16,8 +16,8 @@ from pylint.checkers import BaseChecker from pylint.checkers.utils import ( - is_enum, get_node_first_ancestor_of_type, + is_enum, only_required_for_messages, ) from pylint.typing import MessageDefinitionTuple From 92e135dd725f299854a5234b4a95e49cb62679af Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 5 Aug 2023 15:43:16 -0400 Subject: [PATCH 10/14] See if we can revert this change --- pylint/checkers/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylint/checkers/utils.py b/pylint/checkers/utils.py index ac33e93ec0..edb0aa6c3b 100644 --- a/pylint/checkers/utils.py +++ b/pylint/checkers/utils.py @@ -1471,7 +1471,7 @@ def node_type(node: nodes.NodeNG) -> SuccessfulInferenceResult | None: # don't handle it for now types: set[SuccessfulInferenceResult] = set() try: - for var_type in node.infer(context=InferenceContext()): + for var_type in node.infer(): if isinstance(var_type, util.UninferableBase) or is_none(var_type): continue types.add(var_type) From a06f9ca64668b612151d9b3fdd5f9cdb4e782b5b Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 5 Aug 2023 15:44:35 -0400 Subject: [PATCH 11/14] Remove unused code --- pylint/checkers/typecheck.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/pylint/checkers/typecheck.py b/pylint/checkers/typecheck.py index 2cef6fc75b..06748f6bd9 100644 --- a/pylint/checkers/typecheck.py +++ b/pylint/checkers/typecheck.py @@ -12,8 +12,7 @@ import re import shlex import sys -import types -from collections.abc import Callable, Iterable, Iterator, Sequence +from collections.abc import Callable, Iterable from functools import cached_property, singledispatch from re import Pattern from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union @@ -64,8 +63,6 @@ nodes.ClassDef, ] -_T = TypeVar("_T") - STR_FORMAT = {"builtins.str.format"} ASYNCIO_COROUTINE = "asyncio.coroutines.coroutine" BUILTIN_TUPLE = "builtins.tuple" @@ -105,24 +102,6 @@ class VERSION_COMPATIBLE_OVERLOAD: VERSION_COMPATIBLE_OVERLOAD_SENTINEL = VERSION_COMPATIBLE_OVERLOAD() -def _unflatten(iterable: Iterable[_T]) -> Iterator[_T]: - for index, elem in enumerate(iterable): - if isinstance(elem, Sequence) and not isinstance(elem, str): - yield from _unflatten(elem) - elif elem and not index: - # We're interested only in the first element. - yield elem # type: ignore[misc] - - -def _flatten_container(iterable: Iterable[_T]) -> Iterator[_T]: - # Flatten nested containers into a single iterable - for item in iterable: - if isinstance(item, (list, tuple, types.GeneratorType)): - yield from _flatten_container(item) - else: - yield item - - def _is_owner_ignored( owner: SuccessfulInferenceResult, attrname: str | None, From 0257686398d328ed250553526e0b2e9269a09b98 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 5 Aug 2023 19:45:19 +0000 Subject: [PATCH 12/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pylint/checkers/typecheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylint/checkers/typecheck.py b/pylint/checkers/typecheck.py index 06748f6bd9..aff1ceceac 100644 --- a/pylint/checkers/typecheck.py +++ b/pylint/checkers/typecheck.py @@ -15,7 +15,7 @@ from collections.abc import Callable, Iterable from functools import cached_property, singledispatch from re import Pattern -from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union +from typing import TYPE_CHECKING, Any, Literal, Union import astroid import astroid.exceptions From dc66fc3abdd4c003ed45b1e7821a098ac94ba1fb Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 5 Aug 2023 15:52:07 -0400 Subject: [PATCH 13/14] Use root() --- pylint/checkers/design_analysis.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py index 4748828a09..da4ae9c7e9 100644 --- a/pylint/checkers/design_analysis.py +++ b/pylint/checkers/design_analysis.py @@ -15,11 +15,7 @@ from astroid import nodes from pylint.checkers import BaseChecker -from pylint.checkers.utils import ( - get_node_first_ancestor_of_type, - is_enum, - only_required_for_messages, -) +from pylint.checkers.utils import is_enum, only_required_for_messages from pylint.typing import MessageDefinitionTuple if TYPE_CHECKING: @@ -449,11 +445,11 @@ def visit_classdef(self, node: nodes.ClassDef) -> None: # results, mutating instance_attrs can become a real mess. Filter # them out here until the root cause is solved. # https://github.com/pylint-dev/astroid/issues/2273 - this_module = get_node_first_ancestor_of_type(node, nodes.Module) + root = node.root() filtered_attrs = [ k for (k, v) in node.instance_attrs.items() - if get_node_first_ancestor_of_type(v[0], nodes.Module) is this_module + if v[0].root() is root ] if len(filtered_attrs) > self.linter.config.max_attributes: self.add_message( From c82376daba50fe03eab3b5d5c88e9d9d592bb6de Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 5 Aug 2023 19:52:51 +0000 Subject: [PATCH 14/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pylint/checkers/design_analysis.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py index da4ae9c7e9..f8c1d5c6c9 100644 --- a/pylint/checkers/design_analysis.py +++ b/pylint/checkers/design_analysis.py @@ -447,9 +447,7 @@ def visit_classdef(self, node: nodes.ClassDef) -> None: # https://github.com/pylint-dev/astroid/issues/2273 root = node.root() filtered_attrs = [ - k - for (k, v) in node.instance_attrs.items() - if v[0].root() is root + k for (k, v) in node.instance_attrs.items() if v[0].root() is root ] if len(filtered_attrs) > self.linter.config.max_attributes: self.add_message(