Lint all files in a directory by expanding arguments (#5682)

* Added  --recursive=y/n option and a mention in FAQ and user guide

Co-authored-by: Daniël van Noord <[email protected]>
Co-authored-by: Jacob Walls <[email protected]>
Co-authored-by: Pierre Sassoulas <[email protected]>

Author: 4 people

ChangeLog

@@ -9,6 +9,12 @@ Release date: TBA
 ..
   Put new features here and also in 'doc/whatsnew/2.13.rst'
 
+* Add ``--recursive`` option to allow recursive discovery of all modules and packages in subtree. Running pylint with
+  ``--recursive=y`` option will check all discovered ``.py`` files and packages found inside subtree of directory provided
+  as parameter to pylint.
+
+  Closes #352
+
 * Add ``modified-iterating-list``, ``modified-iterating-dict`` and ``modified-iterating-set``,
   emitted when items are added to or removed from respectively a list, dictionary or
   set being iterated through.

doc/faq.rst

@@ -116,6 +116,27 @@ For example::
 
 Much probably. Read :ref:`ide-integration`
 
+3.5 I need to run pylint over all modules and packages in my project directory.
+-------------------------------------------------------------------------------
+
+By default the ``pylint`` command only accepts a list of python modules and packages. Using a
+directory which is not a package results in an error::
+
+    pylint mydir
+    ************* Module mydir
+    mydir/__init__.py:1:0: F0010: error while code parsing: Unable to load file mydir/__init__.py:
+    [Errno 2] No such file or directory: 'mydir/__init__.py' (parse-error)
+
+To execute pylint over all modules and packages under the directory, the ``--recursive=y`` option must
+be provided. This option makes ``pylint`` attempt to discover all modules (files ending with ``.py`` extension)
+and all packages (all directories containing a ``__init__.py`` file).
+Those modules and packages are then analyzed::
+
+    pylint --recursive=y mydir
+
+When ``--recursive=y`` option is used, modules and packages are also accepted as parameters::
+
+    pylint --recursive=y mydir mymodule mypackage
 
 4. Message Control
 ==================

doc/user_guide/run.rst

@@ -33,6 +33,10 @@ will work if ``directory`` is a python package (i.e. has an __init__.py
 file or it is an implicit namespace package) or if "directory" is in the
 python path.
 
+By default, pylint will exit with an error when one of the arguments is a directory which is not
+a python package. In order to run pylint over all modules and packages within the provided
+subtree of a directory, the ``--recursive=y`` option must be provided.
+
 For more details on this see the :ref:`faq`.
 
 It is also possible to call Pylint from another Python program,

doc/whatsnew/2.13.rst

@@ -92,6 +92,12 @@ Extensions
 Other Changes
 =============
 
+* Add ``--recursive`` option to allow recursive discovery of all modules and packages in subtree. Running pylint with
+  ``--recursive=y`` option will check all discovered ``.py`` files and packages found inside subtree of directory provided
+  as parameter to pylint.
+
+  Closes #352
+
 * Fix false-negative for ``assignment-from-none`` checker with list.sort() method.
 
   Closes #5722

pylint/lint/pylinter.py

@@ -515,6 +515,15 @@ def make_options() -> Tuple[Tuple[str, OptionDict], ...]:
                     ),
                 },
             ),
+            (
+                "recursive",
+                {
+                    "type": "yn",
+                    "metavar": "<yn>",
+                    "default": False,
+                    "help": "Discover python modules and packages in the file system subtree.",
+                },
+            ),
             (
                 "py-version",
                 {
@@ -1005,6 +1014,33 @@ def initialize(self):
             if not msg.may_be_emitted():
                 self._msgs_state[msg.msgid] = False
 
+    @staticmethod
+    def _discover_files(files_or_modules: Sequence[str]) -> Iterator[str]:
+        """Discover python modules and packages in subdirectory.
+
+        Returns iterator of paths to discovered modules and packages.
+        """
+        for something in files_or_modules:
+            if os.path.isdir(something) and not os.path.isfile(
+                os.path.join(something, "__init__.py")
+            ):
+                skip_subtrees: List[str] = []
+                for root, _, files in os.walk(something):
+                    if any(root.startswith(s) for s in skip_subtrees):
+                        # Skip subtree of already discovered package.
+                        continue
+                    if "__init__.py" in files:
+                        skip_subtrees.append(root)
+                        yield root
+                    else:
+                        yield from (
+                            os.path.join(root, file)
+                            for file in files
+                            if file.endswith(".py")
+                        )
+            else:
+                yield something
+
     def check(self, files_or_modules: Union[Sequence[str], str]) -> None:
         """main checking entry: check a list of files or modules from their name.
 
@@ -1019,6 +1055,8 @@ def check(self, files_or_modules: Union[Sequence[str], str]) -> None:
                 DeprecationWarning,
             )
             files_or_modules = (files_or_modules,)  # type: ignore[assignment]
+        if self.config.recursive:
+            files_or_modules = tuple(self._discover_files(files_or_modules))
         if self.config.from_stdin:
             if len(files_or_modules) != 1:
                 raise exceptions.InvalidArgsError(

tests/regrtest_data/directory/package/__init__.py

@@ -100,6 +100,24 @@ def _configure_lc_ctype(lc_ctype: str) -> Iterator:
             os.environ[lc_ctype_env] = original_lctype
 
 
+@contextlib.contextmanager
+def _test_sys_path() -> Generator[None, None, None]:
+    original_path = sys.path
+    try:
+        yield
+    finally:
+        sys.path = original_path
+
+
+@contextlib.contextmanager
+def _test_cwd() -> Generator[None, None, None]:
+    original_dir = os.getcwd()
+    try:
+        yield
+    finally:
+        os.chdir(original_dir)
+
+
 class MultiReporter(BaseReporter):
     def __init__(self, reporters: List[BaseReporter]) -> None:
         # pylint: disable=super-init-not-called
@@ -810,14 +828,6 @@ def test_fail_on_edge_case(self, opts, out):
 
     @staticmethod
     def test_modify_sys_path() -> None:
-        @contextlib.contextmanager
-        def test_sys_path() -> Generator[None, None, None]:
-            original_path = sys.path
-            try:
-                yield
-            finally:
-                sys.path = original_path
-
         @contextlib.contextmanager
         def test_environ_pythonpath(
             new_pythonpath: Optional[str],
@@ -837,7 +847,7 @@ def test_environ_pythonpath(
                     # Only delete PYTHONPATH if new_pythonpath wasn't None
                     del os.environ["PYTHONPATH"]
 
-        with test_sys_path(), patch("os.getcwd") as mock_getcwd:
+        with _test_sys_path(), patch("os.getcwd") as mock_getcwd:
             cwd = "/tmp/pytest-of-root/pytest-0/test_do_not_import_files_from_0"
             mock_getcwd.return_value = cwd
             default_paths = [
@@ -1284,3 +1294,47 @@ def test_regex_paths_csv_validator() -> None:
         with pytest.raises(SystemExit) as ex:
             Run(["--ignore-paths", "test", join(HERE, "regrtest_data", "empty.py")])
         assert ex.value.code == 0
+
+    def test_regression_recursive(self):
+        self._test_output(
+            [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=n"],
+            expected_output="No such file or directory",
+        )
+
+    def test_recursive(self):
+        self._runtest(
+            [join(HERE, "regrtest_data", "directory", "subdirectory"), "--recursive=y"],
+            code=0,
+        )
+
+    def test_recursive_current_dir(self):
+        with _test_sys_path():
+            # pytest is including directory HERE/regrtest_data to sys.path which causes
+            # astroid to believe that directory is a package.
+            sys.path = [
+                path
+                for path in sys.path
+                if not os.path.basename(path) == "regrtest_data"
+            ]
+            with _test_cwd():
+                os.chdir(join(HERE, "regrtest_data", "directory"))
+                self._runtest(
+                    [".", "--recursive=y"],
+                    code=0,
+                )
+
+    def test_regression_recursive_current_dir(self):
+        with _test_sys_path():
+            # pytest is including directory HERE/regrtest_data to sys.path which causes
+            # astroid to believe that directory is a package.
+            sys.path = [
+                path
+                for path in sys.path
+                if not os.path.basename(path) == "regrtest_data"
+            ]
+            with _test_cwd():
+                os.chdir(join(HERE, "regrtest_data", "directory"))
+                self._test_output(
+                    ["."],
+                    expected_output="No such file or directory",
+                )