🍒[libclang] [Dependency Scanning] Diagnosing Out-Of-Date File System Cache Entries

clang/include/clang-c/Dependencies.h

@@ -785,6 +785,103 @@ CINDEX_LINKAGE CXString clang_experimental_DepGraphModuleLinkLibrary_getLibrary(
 CINDEX_LINKAGE bool clang_experimental_DepGraphModuleLinkLibrary_isFramework(
     CXDepGraphModuleLinkLibrary);
 
+/**
+ * The kind of scanning file system cache out-of-date entries.
+ */
+typedef enum {
+  /**
+   * The entry is negatively stat cached (which indicates the file did not exist
+   * the first time it was looked up during scanning), but the cached file
+   * exists on the underlying file system.
+   */
+  NegativelyCached,
+
+  /**
+   * The entry indicates that for the cached file, its cached size
+   * is different from its size reported by the underlying file system.
+   */
+  SizeChanged
+} CXDepScanFSCacheOutOfDateKind;
+
+/**
+ * The opaque object that contains the scanning file system cache's out-of-date
+ * entires.
+ */
+typedef struct CXOpaqueDepScanFSOutOfDateEntrySet *CXDepScanFSOutOfDateEntrySet;
+
+/**
+ * The opaque object that represents a single scanning file system cache's out-
+ * of-date entry.
+ */
+typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry;
+
+/**
+ * Returns all the file system cache out-of-date entries given a
+ * \c CXDependencyScannerService .
+ *
+ * This function is intended to be called when the build has finished,
+ * and the \c CXDependencyScannerService instance is about to be disposed.
+ *
+ * The \c CXDependencyScannerService instance owns the strings used
+ * by the out-of-date entries and should be disposed after the
+ * out-of-date entries are used and disposed.
+ */
+CINDEX_LINKAGE CXDepScanFSOutOfDateEntrySet
+clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
+    CXDependencyScannerService S);
+
+/**
+ * Returns the number of out-of-date entries contained in a
+ * \c CXDepScanFSOutOfDateEntrySet .
+ */
+CINDEX_LINKAGE size_t
+clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+    CXDepScanFSOutOfDateEntrySet Entries);
+
+/**
+ * Returns the out-of-date entry at offset \p Idx of the \c
+ * CXDepScanFSOutOfDateEntrySet instance.
+ */
+CINDEX_LINKAGE CXDepScanFSOutOfDateEntry
+clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(
+    CXDepScanFSOutOfDateEntrySet Entries, size_t Idx);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry, returns its Kind.
+ */
+CINDEX_LINKAGE CXDepScanFSCacheOutOfDateKind
+clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry, returns the path.
+ */
+CINDEX_LINKAGE CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged,
+ * returns the cached size.
+ */
+CINDEX_LINKAGE uint64_t
+clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged,
+ * returns the actual size on the underlying file system.
+ */
+CINDEX_LINKAGE uint64_t
+clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Dispose the \c CXDepScanFSOutOfDateEntrySet instance.
+ */
+CINDEX_LINKAGE void
+clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+    CXDepScanFSOutOfDateEntrySet Entries);
+
 /**
  * @}
  */

clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h

@@ -19,6 +19,7 @@
 #include "llvm/Support/VirtualFileSystem.h"
 #include <mutex>
 #include <optional>
+#include <variant>
 
 namespace clang {
 namespace tooling {
@@ -233,6 +234,35 @@ class DependencyScanningFilesystemSharedCache {
   CacheShard &getShardForFilename(StringRef Filename) const;
   CacheShard &getShardForUID(llvm::sys::fs::UniqueID UID) const;
 
+  struct OutOfDateEntry {
+    // A null terminated string that contains a path.
+    const char *Path = nullptr;
+
+    struct NegativelyCachedInfo {};
+    struct SizeChangedInfo {
+      uint64_t CachedSize = 0;
+      uint64_t ActualSize = 0;
+    };
+
+    std::variant<NegativelyCachedInfo, SizeChangedInfo> Info;
+
+    OutOfDateEntry(const char *Path)
+        : Path(Path), Info(NegativelyCachedInfo{}) {}
+
+    OutOfDateEntry(const char *Path, uint64_t CachedSize, uint64_t ActualSize)
+        : Path(Path), Info(SizeChangedInfo{CachedSize, ActualSize}) {}
+  };
+
+  /// Visits all cached entries and re-stat an entry using UnderlyingFS to check
+  /// if the cache contains out-of-date entries. An entry can be out-of-date for
+  /// two reasons:
+  ///  1. The entry contains a stat error, indicating the file did not exist
+  ///     in the cache, but the file exists on the UnderlyingFS.
+  ///  2. The entry is associated with a file whose size is different from the
+  ///     size of the file on the same path on the UnderlyingFS.
+  std::vector<OutOfDateEntry>
+  getOutOfDateEntries(llvm::vfs::FileSystem &UnderlyingFS) const;
+
 private:
   std::unique_ptr<CacheShard[]> CacheShards;
   unsigned NumShards;

clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

@@ -113,6 +113,50 @@ DependencyScanningFilesystemSharedCache::getShardForUID(
   return CacheShards[Hash % NumShards];
 }
 
+std::vector<DependencyScanningFilesystemSharedCache::OutOfDateEntry>
+DependencyScanningFilesystemSharedCache::getOutOfDateEntries(
+    llvm::vfs::FileSystem &UnderlyingFS) const {
+  // Iterate through all shards and look for cached stat errors.
+  std::vector<OutOfDateEntry> InvalidDiagInfo;
+  for (unsigned i = 0; i < NumShards; i++) {
+    const CacheShard &Shard = CacheShards[i];
+    std::lock_guard<std::mutex> LockGuard(Shard.CacheLock);
+    for (const auto &[Path, CachedPair] : Shard.CacheByFilename) {
+      const CachedFileSystemEntry *Entry = CachedPair.first;
+      llvm::ErrorOr<llvm::vfs::Status> Status = UnderlyingFS.status(Path);
+      if (Status) {
+        if (Entry->getError()) {
+          // This is the case where we have cached the non-existence
+          // of the file at Path first, and a file at the path is created
+          // later. The cache entry is not invalidated (as we have no good
+          // way to do it now), which may lead to missing file build errors.
+          InvalidDiagInfo.emplace_back(Path.data());
+        } else {
+          llvm::vfs::Status CachedStatus = Entry->getStatus();
+          if (Status->getType() == llvm::sys::fs::file_type::regular_file &&
+              Status->getType() == CachedStatus.getType()) {
+            // We only check regular files. Directory files sizes could change
+            // due to content changes, and reporting directory size changes can
+            // lead to false positives.
+            // TODO: At the moment, we do not detect symlinks to files whose
+            // size may change. We need to decide if we want to detect cached
+            // symlink size changes. We can also expand this to detect file
+            // type changes.
+            uint64_t CachedSize = CachedStatus.getSize();
+            uint64_t ActualSize = Status->getSize();
+            if (CachedSize != ActualSize) {
+              // This is the case where the cached file has a different size
+              // from the actual file that comes from the underlying FS.
+              InvalidDiagInfo.emplace_back(Path.data(), CachedSize, ActualSize);
+            }
+          }
+        }
+      }
+    }
+  }
+  return InvalidDiagInfo;
+}
+
 const CachedFileSystemEntry *
 DependencyScanningFilesystemSharedCache::CacheShard::findEntryByFilename(
     StringRef Filename) const {

clang/test/ClangScanDeps/error-c-api.cpp

@@ -4,3 +4,4 @@
 
 // CHECK: error: failed to get dependencies
 // CHECK-NEXT: 'missing.h' file not found
+// CHECK-NEXT: number of out of date file system cache entries: 0

clang/tools/c-index-test/core_main.cpp

@@ -916,6 +916,20 @@ static int scanDeps(ArrayRef<const char *> Args, std::string WorkingDirectory,
     clang_disposeString(Spelling);
     clang_disposeDiagnostic(Diag);
   }
+
+  CXDepScanFSOutOfDateEntrySet OutOfDateEntrySet =
+      clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
+          Service);
+
+  llvm::errs()
+      << "note: number of out of date file system cache entries: "
+      << clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+             OutOfDateEntrySet)
+      << "\n";
+
+  clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+      OutOfDateEntrySet);
+
   return 1;
 }
 

clang/tools/libclang/CDependencies.cpp

@@ -45,7 +45,6 @@ struct DependencyScannerServiceOptions {
 
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerServiceOptions,
                                    CXDependencyScannerServiceOptions)
-
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService,
                                    CXDependencyScannerService)
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker,
@@ -816,3 +815,96 @@ bool clang_experimental_DepGraphModuleLinkLibrary_isFramework(
   return Lib->IsFramework;
 }
 
+
+namespace {
+typedef std::vector<DependencyScanningFilesystemSharedCache::OutOfDateEntry>
+    DependencyScannerFSOutOfDateEntrySet;
+
+typedef DependencyScanningFilesystemSharedCache::OutOfDateEntry
+    DependencyScannerFSOutOfDateEntry;
+} // namespace
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntrySet,
+                                   CXDepScanFSOutOfDateEntrySet)
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry,
+                                   CXDepScanFSOutOfDateEntry)
+
+CXDepScanFSOutOfDateEntrySet
+clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
+    CXDependencyScannerService S) {
+  DependencyScanningService &Service = *unwrap(S);
+
+  // FIXME: CAS FS currently does not use the shared cache, and cannot produce
+  // the same diagnostics. We should add such a diagnostics to CAS as well.
+  if (Service.useCASFS())
+    return nullptr;
+
+  // Note that it is critical that this FS is the same as the default virtual
+  // file system we pass to the DependencyScanningWorkers.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
+      llvm::vfs::createPhysicalFileSystem();
+
+  DependencyScannerFSOutOfDateEntrySet *OODEntrySet =
+      new DependencyScannerFSOutOfDateEntrySet();
+  *OODEntrySet = Service.getSharedCache().getOutOfDateEntries(*FS);
+
+  return wrap(OODEntrySet);
+}
+
+size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+    CXDepScanFSOutOfDateEntrySet Entries) {
+  return unwrap(Entries)->size();
+}
+
+CXDepScanFSOutOfDateEntry
+clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(
+    CXDepScanFSOutOfDateEntrySet Entries, size_t Idx) {
+  DependencyScannerFSOutOfDateEntrySet *EntSet = unwrap(Entries);
+  return wrap(&(*EntSet)[Idx]);
+}
+
+CXDepScanFSCacheOutOfDateKind
+clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  auto &Info = E->Info;
+  return std::visit(
+      llvm::makeVisitor(
+          [](const DependencyScannerFSOutOfDateEntry::NegativelyCachedInfo
+                 &Info) { return NegativelyCached; },
+          [](const DependencyScannerFSOutOfDateEntry::SizeChangedInfo &Info) {
+            return SizeChanged;
+          }),
+      Info);
+}
+
+CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(
+    CXDepScanFSOutOfDateEntry Entry) {
+  return cxstring::createRef(unwrap(Entry)->Path);
+}
+
+static DependencyScannerFSOutOfDateEntry::SizeChangedInfo *
+getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) {
+  auto *SizeInfo =
+      std::get_if<DependencyScannerFSOutOfDateEntry::SizeChangedInfo>(&E->Info);
+  assert(SizeInfo && "Wrong entry kind to get size changed info!");
+  return SizeInfo;
+}
+
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  return getOutOfDateEntrySizeChangedInfo(E)->CachedSize;
+}
+
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  return getOutOfDateEntrySizeChangedInfo(E)->ActualSize;
+}
+
+void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+    CXDepScanFSOutOfDateEntrySet Entries) {
+  delete unwrap(Entries);
+}
+

clang/tools/libclang/libclang.map

@@ -580,6 +580,14 @@ LLVM_21 {
     clang_experimental_DepGraphModuleLinkLibrarySet_getLinkLibrary;
     clang_experimental_DepGraphModuleLinkLibrary_getLibrary;
     clang_experimental_DepGraphModuleLinkLibrary_isFramework;
+    clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getKind;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getPath;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet;
 };
 
 # Example of how to add a new symbol version entry.  If you do add a new symbol