diff --git a/mypy/build.py b/mypy/build.py
index 52bee8d1ac48..df3b57945dba 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -2369,6 +2369,12 @@ def find_module_and_diagnose(manager: BuildManager,
             raise CompileError(["mypy: can't find module '%s'" % id])
 
 
+def find_module_simple(id: str, manager: BuildManager) -> Optional[str]:
+    """Find a filesystem path for module `id` or `None` if not found."""
+    return manager.find_module_cache.find_module(id, manager.search_paths,
+                                                 manager.options.python_executable)
+
+
 def in_partial_package(id: str, manager: BuildManager) -> bool:
     """Check if a missing module can potentially be a part of a package.
 
@@ -2643,9 +2649,17 @@ def load_graph(sources: List[BuildSource], manager: BuildManager,
         #   (since direct dependencies reflect the imports found in the source)
         #   but A's cached *indirect* dependency on C is wrong.
         dependencies = [dep for dep in st.dependencies if st.priorities.get(dep) != PRI_INDIRECT]
-        added = [dep for dep in st.suppressed
-                 if manager.find_module_cache.find_module(dep, manager.search_paths,
-                                                          manager.options.python_executable)]
+        if not manager.use_fine_grained_cache():
+            # TODO: Ideally we could skip here modules that appeared in st.suppressed
+            # because they are not in build with `follow-imports=skip`.
+            # This way we could avoid overhead of cloning options in `State.__init__()`
+            # below to get the option value. This is quite minor performance loss however.
+            added = [dep for dep in st.suppressed if find_module_simple(dep, manager)]
+        else:
+            # During initial loading we don't care about newly added modules,
+            # they will be taken care of during fine grained update. See also
+            # comment about this in `State.__init__()`.
+            added = []
         for dep in st.ancestors + dependencies + st.suppressed:
             # We don't want to recheck imports marked with '# type: ignore'
             # so we ignore any suppressed module not explicitly re-included
diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test
index dda28ed72fd1..8b181f22e8ff 100644
--- a/test-data/unit/fine-grained-modules.test
+++ b/test-data/unit/fine-grained-modules.test
@@ -1947,3 +1947,181 @@ from b.foo import bar
 x = '10'
 [out]
 ==
+
+[case testFineAddedMissingStubs]
+# flags: --ignore-missing-imports
+from missing import f
+f(int())
+[file missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineAddedMissingStubsPackage]
+# flags: --ignore-missing-imports
+import package.missing
+package.missing.f(int())
+[file package/__init__.pyi.2]
+[file package/missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineAddedMissingStubsPackageFrom]
+# flags: --ignore-missing-imports
+from package import missing
+missing.f(int())
+[file package/__init__.pyi.2]
+[file package/missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineAddedMissingStubsPackagePartial]
+# flags: --ignore-missing-imports
+import package.missing
+package.missing.f(int())
+[file package/__init__.pyi]
+[file package/missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:3: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineAddedMissingStubsPackagePartialGetAttr]
+import package.missing
+package.missing.f(int())
+[file package/__init__.pyi]
+from typing import Any
+def __getattr__(attr: str) -> Any: ...
+[file package/missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineAddedMissingStubsIgnore]
+from missing import f  # type: ignore
+f(int())
+[file missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineAddedMissingStubsIgnorePackage]
+import package.missing  # type: ignore
+package.missing.f(int())
+[file package/__init__.pyi.2]
+[file package/missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineAddedMissingStubsIgnorePackageFrom]
+from package import missing  # type: ignore
+missing.f(int())
+[file package/__init__.pyi.2]
+[file package/missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineAddedMissingStubsIgnorePackagePartial]
+import package.missing  # type: ignore
+package.missing.f(int())
+[file package/__init__.pyi]
+[file package/missing.pyi.2]
+def f(x: str) -> None: pass
+[out]
+==
+main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testFineFollowImportSkipNotInvalidatedOnPresent]
+# flags: --follow-imports=skip
+# cmd: mypy main.py
+[file main.py]
+import other
+[file other.py]
+x = 1
+[file other.py.2]
+x = 'hi'
+[stale]
+[rechecked]
+[out]
+==
+
+[case testFineFollowImportSkipNotInvalidatedOnPresentPackage]
+# flags: --follow-imports=skip
+# cmd: mypy main.py
+[file main.py]
+import other
+[file other/__init__.py]
+x = 1
+[file other/__init__.py.2]
+x = 'hi'
+[stale]
+[rechecked]
+[out]
+==
+
+[case testFineFollowImportSkipNotInvalidatedOnAdded]
+# flags: --follow-imports=skip --ignore-missing-imports
+# cmd: mypy main.py
+[file main.py]
+import other
+[file other.py.2]
+x = 1
+[stale]
+[rechecked]
+[out]
+==
+
+-- TODO: Fix this: stubs should be followed normally even with follow-imports=skip
+[case testFineFollowImportSkipInvalidatedOnAddedStub-skip]
+# flags: --follow-imports=skip --ignore-missing-imports
+# cmd: mypy main.py
+[file main.py]
+import other
+x: str = other.x
+[file other.pyi.2]
+x = 1
+[stale main, other]
+[rechecked main, other]
+[out]
+==
+main:2: error: Incompatible types in assignment (expression has type "int", variable has type "str")
+
+[case testFineFollowImportSkipNotInvalidatedOnAddedStubOnFollowForStubs]
+# flags: --follow-imports=skip --ignore-missing-imports --config-file=tmp/mypy.ini
+# cmd: mypy main.py
+[file main.py]
+import other
+[file other.pyi.2]
+x = 1
+[file mypy.ini]
+[[mypy]
+follow_imports_for_stubs = True
+[stale]
+[rechecked]
+[out]
+==
+
+[case testFineAddedSkippedStubsPackageFrom]
+# flags: --follow-imports=skip --ignore-missing-imports
+# cmd: mypy main.py
+# cmd2: mypy main.py package/__init__.py package/missing.py
+[file main.py]
+from package import missing
+missing.f(int())
+[file package/__init__.py]
+[file package/missing.py]
+def f(x: str) -> None: pass
+[out]
+==
+main.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"