Various fixes to following imports in mypy daemon (#8590)

Included fixes:

- filter out bogus suppressed modules to speed things up
- fix issue with import inside function
- fix stub packages
- also support `/` path separators on Windows
- update search path based on paths given on the command line
- support changes to files passed on the command line
- add logging
- minimal namespace package support

Namespace package support is not complete, but the issue I found isn't
specific to following imports in mypy daemon, so I didn't try to fix it
in this PR. I'll create a follow-up issue.

Author: JukkaL

mypy/dmypy_server.py

@@ -423,7 +423,7 @@ def initialize_fine_grained(self, sources: List[BuildSource],
         self.fine_grained_manager = FineGrainedBuildManager(result)
 
         if self.following_imports():
-            sources = find_all_sources_in_build(self.fine_grained_manager.graph)
+            sources = find_all_sources_in_build(self.fine_grained_manager.graph, sources)
             self.update_sources(sources)
 
         self.previous_sources = sources
@@ -523,13 +523,9 @@ def fine_grained_increment(self,
 
     def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> List[str]:
         """Like fine_grained_increment, but follow imports."""
+        t0 = time.time()
 
-        # TODO:
-        #  - file events
-        #  - search path updates
-        #  - logging
-
-        changed_paths = self.fswatcher.find_changed()
+        # TODO: Support file events
 
         assert self.fine_grained_manager is not None
         fine_grained_manager = self.fine_grained_manager
@@ -538,9 +534,13 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L
 
         orig_modules = list(graph.keys())
 
-        # TODO: Are the differences from fine_grained_increment(), such as
-        #       updating sources after finding changed, necessary?
+        # TODO: Are the differences from fine_grained_increment(), necessary?
         self.update_sources(sources)
+        changed_paths = self.fswatcher.find_changed()
+        manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir)
+
+        t1 = time.time()
+        manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0))
 
         sources_set = {source.module for source in sources}
 
@@ -570,10 +570,14 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L
             # TODO: Removed?
             worklist.extend(changed)
 
+        t2 = time.time()
+
         for module_id, state in graph.items():
             refresh_suppressed_submodules(module_id, state.path, fine_grained_manager.deps, graph,
                                           self.fscache)
 
+        t3 = time.time()
+
         # There may be new files that became available, currently treated as
         # suppressed imports. Process them.
         seen_suppressed = set()  # type: Set[str]
@@ -593,6 +597,8 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L
                 refresh_suppressed_submodules(module_id, path, fine_grained_manager.deps, graph,
                                               self.fscache)
 
+        t4 = time.time()
+
         # Find all original modules in graph that were not reached -- they are deleted.
         to_delete = []
         for module_id in orig_modules:
@@ -612,6 +618,17 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L
 
         self.previous_sources = find_all_sources_in_build(graph)
         self.update_sources(self.previous_sources)
+
+        t5 = time.time()
+
+        manager.log("fine-grained increment: update: {:.3f}s".format(t5 - t1))
+        manager.add_stats(
+            find_changes_time=t1 - t0,
+            fg_update_time=t2 - t1,
+            refresh_suppressed_time=t3 - t2,
+            find_added_supressed_time=t4 - t3,
+            cleanup_time=t5 - t4)
+
         return messages
 
     def follow_imports(self,
@@ -671,6 +688,10 @@ def find_added_suppressed(self,
         for module, state in graph.items():
             all_suppressed |= state.suppressed_set
 
+        # Filter out things that shouldn't actually be considered suppressed.
+        # TODO: Figure out why these are treated as suppressed
+        all_suppressed = {module for module in all_suppressed if module not in graph}
+
         # TODO: Namespace packages
         # TODO: Handle seen?
 
@@ -720,6 +741,9 @@ def pretty_messages(self, messages: List[str], n_sources: int,
 
     def update_sources(self, sources: List[BuildSource]) -> None:
         paths = [source.path for source in sources if source.path is not None]
+        if self.following_imports():
+            # Filter out directories (used for namespace packages).
+            paths = [path for path in paths if self.fscache.isfile(path)]
         self.fswatcher.add_watched_paths(paths)
 
     def update_changed(self,
@@ -828,10 +852,13 @@ def get_meminfo() -> Dict[str, Any]:
     return res
 
 
-def find_all_sources_in_build(graph: mypy.build.Graph) -> List[BuildSource]:
-    result = []
+def find_all_sources_in_build(graph: mypy.build.Graph,
+                              extra: Sequence[BuildSource] = ()) -> List[BuildSource]:
+    result = list(extra)
+    seen = set(source.module for source in result)
     for module, state in graph.items():
-        result.append(BuildSource(state.path, module))
+        if module not in seen:
+            result.append(BuildSource(state.path, module))
     return result
 
 

mypy/modulefinder.py

@@ -77,9 +77,11 @@ def __init__(self, path: Optional[str], module: Optional[str],
         self.base_dir = base_dir  # Directory where the package is rooted (e.g. 'xxx/yyy')
 
     def __repr__(self) -> str:
-        return '<BuildSource path=%r module=%r has_text=%s>' % (self.path,
-                                                                self.module,
-                                                                self.text is not None)
+        return '<BuildSource path=%r module=%r has_text=%s base_dir=%s>' % (
+            self.path,
+            self.module,
+            self.text is not None,
+            self.base_dir)
 
 
 class FindModuleCache:

mypy/server/update.py

@@ -113,6 +113,7 @@
 """
 
 import os
+import sys
 import time
 from typing import (
     Dict, List, Set, Tuple, Union, Optional, NamedTuple, Sequence
@@ -1111,6 +1112,17 @@ def target_from_node(module: str,
             return '%s.%s' % (module, node.name)
 
 
+if sys.platform != 'win32':
+    INIT_SUFFIXES = ('/__init__.py', '/__init__.pyi')  # type: Final
+else:
+    INIT_SUFFIXES = (
+        os.sep + '__init__.py',
+        os.sep + '__init__.pyi',
+        os.altsep + '__init__.py',
+        os.altsep + '__init__.pyi',
+    )  # type: Final
+
+
 def refresh_suppressed_submodules(
         module: str,
         path: Optional[str],
@@ -1129,8 +1141,7 @@ def refresh_suppressed_submodules(
         module: target package in which to look for submodules
         path: path of the module
     """
-    # TODO: Windows paths
-    if path is None or not path.endswith(os.sep + '__init__.py'):
+    if path is None or not path.endswith(INIT_SUFFIXES):
         # Only packages have submodules.
         return
     # Find any submodules present in the directory.
@@ -1145,8 +1156,13 @@ def refresh_suppressed_submodules(
         trigger = make_trigger(submodule)
         if trigger in deps:
             for dep in deps[trigger]:
-                # TODO: <...> deps, imports in functions, etc.
+                # TODO: <...> deps, etc.
                 state = graph.get(dep)
+                if not state:
+                    # Maybe it's a non-top-level target. We only care about the module.
+                    dep_module = module_prefix(graph, dep)
+                    if dep_module is not None:
+                        state = graph.get(dep_module)
                 if state:
                     tree = state.tree
                     assert tree  # TODO: What if doesn't exist?

mypy/test/testcheck.py

@@ -336,7 +336,7 @@ def parse_module(self,
             cache = FindModuleCache(search_paths)
             for module_name in module_names.split(' '):
                 path = cache.find_module(module_name)
-                assert isinstance(path, str), "Can't find ad hoc case file"
+                assert isinstance(path, str), "Can't find ad hoc case file: %s" % module_name
                 with open(path, encoding='utf8') as f:
                     program_text = f.read()
                 out.append((module_name, path, program_text))

test-data/unit/fine-grained-follow-imports.test

@@ -542,3 +542,161 @@ main.py:3: error: Too few arguments for "f"
 ==
 main.py:3: error: Too few arguments for "f"
 main.py:4: error: Too many arguments for "f"
+
+[case testFollowImportsNormalPackageInitFile4-only_when_cache]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+
+[file main.py]
+import p1.m  # type: ignore
+from p2 import m  # type: ignore
+
+[file p1/__init__.py.2]
+1()
+
+[file p1/m.py.2]
+''()
+
+[file p2/__init__.py.3]
+''()
+
+[file p2/m.py.3]
+
+[out]
+==
+p1/__init__.py:1: error: "int" not callable
+p1/m.py:1: error: "str" not callable
+==
+p1/__init__.py:1: error: "int" not callable
+p1/m.py:1: error: "str" not callable
+p2/__init__.py:1: error: "str" not callable
+
+[case testFollowImportsNormalSubmoduleCreatedWithImportInFunction]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+
+[file main.py]
+def f() -> None:
+    from p import m
+
+[file p/__init__.py.2]
+1()
+
+[file p/m.py.2]
+''()
+
+[out]
+main.py:2: error: Cannot find implementation or library stub for module named 'p'
+main.py:2: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports
+==
+p/m.py:1: error: "str" not callable
+p/__init__.py:1: error: "int" not callable
+
+[case testFollowImportsNormalPackageInitFileStub]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+
+[file main.py]
+from p import m
+
+[file p/__init__.pyi.2]
+1()
+
+[file p/m.pyi.2]
+''()
+
+[file p/mm.pyi.3]
+x x x
+
+[out]
+main.py:1: error: Cannot find implementation or library stub for module named 'p'
+main.py:1: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports
+==
+p/m.pyi:1: error: "str" not callable
+p/__init__.pyi:1: error: "int" not callable
+==
+p/m.pyi:1: error: "str" not callable
+p/__init__.pyi:1: error: "int" not callable
+
+[case testFollowImportsNormalNamespacePackages]
+# flags: --follow-imports=normal --namespace-packages
+# cmd: mypy main.py
+
+[file main.py]
+import p1.m1
+import p2.m2
+
+[file p1/m1.py]
+1()
+
+[file p2/m2.py.2]
+''()
+
+[delete p2/m2.py.3]
+
+[out]
+p1/m1.py:1: error: "int" not callable
+main.py:2: error: Cannot find implementation or library stub for module named 'p2.m2'
+main.py:2: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports
+==
+p2/m2.py:1: error: "str" not callable
+p1/m1.py:1: error: "int" not callable
+==
+main.py:2: error: Cannot find implementation or library stub for module named 'p2.m2'
+main.py:2: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports
+p1/m1.py:1: error: "int" not callable
+
+[case testFollowImportsNormalNewFileOnCommandLine]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy main.py x.py
+
+[file main.py]
+1()
+
+[file x.py.2]
+''()
+
+[out]
+main.py:1: error: "int" not callable
+==
+x.py:1: error: "str" not callable
+main.py:1: error: "int" not callable
+
+[case testFollowImportsNormalSearchPathUpdate-only_when_nocache]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy main.py src/foo.py
+
+[file main.py]
+
+[file src/foo.py.2]
+import bar
+''()
+
+[file src/bar.py.2]
+1()
+
+[out]
+==
+src/bar.py:1: error: "int" not callable
+src/foo.py:2: error: "str" not callable
+
+[case testFollowImportsNormalSearchPathUpdate-only_when_cache]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy main.py src/foo.py
+
+[file main.py]
+
+[file src/foo.py.2]
+import bar
+''()
+
+[file src/bar.py.2]
+1()
+
+[out]
+==
+src/foo.py:2: error: "str" not callable
+src/bar.py:1: error: "int" not callable