[clang][Modules] Adding C-API for Negative Stat Caching Diagnostics (llvm#10524)

llvm#135703 added a C++ API to the shared cached to diagnose invalid negatively stat cached paths. This PR adds a C API so an external system can take advantage of the diagnostics.

rdar://149147920
(cherry picked from commit b7aa45c)

Author: qiongsiwu

clang/include/clang-c/Dependencies.h

@@ -735,6 +735,11 @@ const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph);
 CINDEX_LINKAGE
 CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph);
 
+CINDEX_LINKAGE
+CXCStringArray
+    clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
+        CXDependencyScannerService);
+
 /**
  * @}
  */

clang/test/ClangScanDeps/error-c-api.cpp

@@ -4,3 +4,4 @@
 
 // CHECK: error: failed to get dependencies
 // CHECK-NEXT: 'missing.h' file not found
+// CHECK-NEXT: number of invalid negatively stat cached paths: 0

clang/tools/c-index-test/core_main.cpp

@@ -894,6 +894,14 @@ static int scanDeps(ArrayRef<const char *> Args, std::string WorkingDirectory,
     clang_disposeString(Spelling);
     clang_disposeDiagnostic(Diag);
   }
+
+  CXCStringArray InvalidNegativeStatCachedPaths =
+      clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
+          Service);
+
+  llvm::errs() << "note: number of invalid negatively stat cached paths: "
+               << InvalidNegativeStatCachedPaths.Count << "\n";
+
   return 1;
 }
 

clang/tools/libclang/CDependencies.cpp

@@ -41,12 +41,57 @@ struct DependencyScannerServiceOptions {
 
   ScanningOutputFormat getFormat() const;
 };
+
+struct CStringsManager {
+  SmallVector<std::unique_ptr<std::vector<const char *>>> OwnedCStr;
+  SmallVector<std::unique_ptr<std::vector<std::string>>> OwnedStdStr;
+
+  /// Doesn't own the string contents.
+  CXCStringArray createCStringsRef(ArrayRef<std::string> Strings) {
+    OwnedCStr.push_back(std::make_unique<std::vector<const char *>>());
+    std::vector<const char *> &CStrings = *OwnedCStr.back();
+    CStrings.reserve(Strings.size());
+    for (const auto &String : Strings)
+      CStrings.push_back(String.c_str());
+    return {CStrings.data(), CStrings.size()};
+  }
+
+  /// Doesn't own the string contents.
+  CXCStringArray createCStringsRef(const llvm::StringSet<> &StringsUnordered) {
+    std::vector<StringRef> Strings;
+
+    for (auto SI = StringsUnordered.begin(), SE = StringsUnordered.end();
+         SI != SE; ++SI)
+      Strings.push_back(SI->getKey());
+
+    llvm::sort(Strings);
+
+    OwnedCStr.push_back(std::make_unique<std::vector<const char *>>());
+    std::vector<const char *> &CStrings = *OwnedCStr.back();
+    CStrings.reserve(Strings.size());
+    for (const auto &String : Strings)
+      CStrings.push_back(String.data());
+    return {CStrings.data(), CStrings.size()};
+  }
+
+  /// Gets ownership of string contents.
+  CXCStringArray createCStringsOwned(std::vector<std::string> &&Strings) {
+    OwnedStdStr.push_back(
+        std::make_unique<std::vector<std::string>>(std::move(Strings)));
+    return createCStringsRef(*OwnedStdStr.back());
+  }
+};
+
+struct DependencyScannerService {
+  DependencyScanningService Service;
+  CStringsManager StrMgr{};
+};
 } // end anonymous namespace
 
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerServiceOptions,
                                    CXDependencyScannerServiceOptions)
 
-DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService,
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerService,
                                    CXDependencyScannerService)
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker,
                                    CXDependencyScannerWorker)
@@ -142,9 +187,9 @@ clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format) {
   // FIXME: Pass default CASOpts and nullptr as CachingOnDiskFileSystem now.
   CASOptions CASOpts;
   IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
-  return wrap(new DependencyScanningService(
+  return wrap(new DependencyScannerService{DependencyScanningService(
       ScanningMode::DependencyDirectivesScan, unwrap(Format), CASOpts,
-      /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS));
+      /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS)});
 }
 
 ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const {
@@ -180,10 +225,10 @@ clang_experimental_DependencyScannerService_create_v1(
     FS = llvm::cantFail(
         llvm::cas::createCachingOnDiskFileSystem(CAS));
   }
-  return wrap(new DependencyScanningService(
+  return wrap(new DependencyScannerService{DependencyScanningService(
       ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts,
       std::move(CAS), std::move(Cache), std::move(FS),
-      unwrap(Opts)->OptimizeArgs));
+      unwrap(Opts)->OptimizeArgs)});
 }
 
 void clang_experimental_DependencyScannerService_dispose_v0(
@@ -213,17 +258,17 @@ void clang_experimental_FileDependenciesList_dispose(
 }
 
 CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0(
-    CXDependencyScannerService Service) {
-  ScanningOutputFormat Format = unwrap(Service)->getFormat();
+    CXDependencyScannerService S) {
+  ScanningOutputFormat Format = unwrap(S)->Service.getFormat();
   bool IsIncludeTreeOutput = Format == ScanningOutputFormat::IncludeTree ||
                              Format == ScanningOutputFormat::FullIncludeTree;
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
       llvm::vfs::createPhysicalFileSystem();
   if (IsIncludeTreeOutput)
-    FS = llvm::cas::createCASProvidingFileSystem(unwrap(Service)->getCAS(),
+    FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->Service.getCAS(),
                                                  std::move(FS));
 
-  return wrap(new DependencyScanningWorker(*unwrap(Service), FS));
+  return wrap(new DependencyScanningWorker(unwrap(S)->Service, FS));
 }
 
 void clang_experimental_DependencyScannerWorker_dispose_v0(
@@ -432,46 +477,6 @@ struct DependencyScannerWorkerScanSettings {
       MLO;
 };
 
-struct CStringsManager {
-  SmallVector<std::unique_ptr<std::vector<const char *>>> OwnedCStr;
-  SmallVector<std::unique_ptr<std::vector<std::string>>> OwnedStdStr;
-
-  /// Doesn't own the string contents.
-  CXCStringArray createCStringsRef(ArrayRef<std::string> Strings) {
-    OwnedCStr.push_back(std::make_unique<std::vector<const char *>>());
-    std::vector<const char *> &CStrings = *OwnedCStr.back();
-    CStrings.reserve(Strings.size());
-    for (const auto &String : Strings)
-      CStrings.push_back(String.c_str());
-    return {CStrings.data(), CStrings.size()};
-  }
-
-  /// Doesn't own the string contents.
-  CXCStringArray createCStringsRef(const llvm::StringSet<> &StringsUnordered) {
-    std::vector<StringRef> Strings;
-
-    for (auto SI = StringsUnordered.begin(), SE = StringsUnordered.end();
-         SI != SE; ++SI)
-      Strings.push_back(SI->getKey());
-
-    llvm::sort(Strings);
-
-    OwnedCStr.push_back(std::make_unique<std::vector<const char *>>());
-    std::vector<const char *> &CStrings = *OwnedCStr.back();
-    CStrings.reserve(Strings.size());
-    for (const auto &String : Strings)
-      CStrings.push_back(String.data());
-    return {CStrings.data(), CStrings.size()};
-  }
-
-  /// Gets ownership of string contents.
-  CXCStringArray createCStringsOwned(std::vector<std::string> &&Strings) {
-    OwnedStdStr.push_back(
-        std::make_unique<std::vector<std::string>>(std::move(Strings)));
-    return createCStringsRef(*OwnedStdStr.back());
-  }
-};
-
 struct DependencyGraph {
   TranslationUnitDeps TUDeps;
   SmallString<256> SerialDiagBuf;
@@ -731,6 +736,38 @@ CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) {
   return unwrap(Graph)->getDiagnosticSet();
 }
 
+CXCStringArray
+clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
+    CXDependencyScannerService S) {
+  DependencyScanningService &Service = unwrap(S)->Service;
+  CStringsManager &StrMgr = unwrap(S)->StrMgr;
+
+  // FIXME: CAS currently does not use the shared cache, and cannot produce
+  // the same diagnostics. We should add such a diagnostics to CAS as well.
+  if (Service.useCASFS())
+    return {nullptr, 0};
+
+  DependencyScanningFilesystemSharedCache &SharedCache =
+      Service.getSharedCache();
+
+  // Note that it is critical that this FS is the same as the default virtual
+  // file system we pass to the DependencyScanningWorkers.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
+      llvm::vfs::createPhysicalFileSystem();
+
+  auto InvaidNegStatCachedPaths =
+      SharedCache.getInvalidNegativeStatCachedPaths(*FS);
+
+  // FIXME: This code here creates copies of strings from
+  // InvaidNegStatCachedPaths. It is acceptable because this C-API is expected
+  // to be called only at the end of a CXDependencyScannerService's lifetime.
+  // In other words, it is called very infrequently. We can change
+  // CStringsManager's interface to accommodate handling arbitrary StringRefs
+  // (which may not be null terminated) if we want to avoid copying.
+  return StrMgr.createCStringsOwned(
+      {InvaidNegStatCachedPaths.begin(), InvaidNegStatCachedPaths.end()});
+}
+
 static std::string
 lookupModuleOutput(const ModuleDeps &MD, ModuleOutputKind MOK, void *MLOContext,
                    std::variant<CXModuleLookupOutputCallback *,

clang/tools/libclang/CXString.cpp

@@ -130,13 +130,13 @@ CXStringSet *createSet(const std::vector<std::string> &Strings) {
 
 CXStringSet *createSet(const llvm::StringSet<> &StringsUnordered) {
   std::vector<StringRef> Strings;
-  
-  for (auto SI = StringsUnordered.begin(),
-            SE = StringsUnordered.end(); SI != SE; ++SI)
+
+  for (auto SI = StringsUnordered.begin(), SE = StringsUnordered.end();
+       SI != SE; ++SI)
     Strings.push_back(SI->getKey());
-  
+
   llvm::sort(Strings);
-  
+
   CXStringSet *Set = new CXStringSet;
   Set->Count = Strings.size();
   Set->Strings = new CXString[Set->Count];

clang/tools/libclang/libclang.map

@@ -575,6 +575,7 @@ LLVM_21 {
     clang_experimental_DependencyScannerServiceOptions_setCWDOptimization;
     clang_experimental_DepGraphModule_isCWDIgnored;
     clang_experimental_DepGraphModule_isInStableDirs; 
+    clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths;
 };
 
 # Example of how to add a new symbol version entry.  If you do add a new symbol