[llvm][cas] Add validate-if-needed to recover from invalid data

Introduce a new validate-if-needed API to the UnifiedOnDiskCache and
llvm-cas tool that triggers out-of-process validation of the CAS once
for every machine boot, and optionally recovers from invalid data by
marking it for garbage collection. This fixes a hole in the CAS data
coherence when a power loss or similar failure causes the OS to not
flush all of the pages in the mmaped on-disk CAS files.

The intent is that clients such as the clang scanning daemon or a build
system should trigger this validation at least once before using the
CAS.

rdar://123542312

Author: benlangmuir

llvm/include/llvm/CAS/BuiltinUnifiedCASDatabases.h

@@ -21,6 +21,39 @@ class ObjectStore;
 Expected<std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>>>
 createOnDiskUnifiedCASDatabases(StringRef Path);
 
+/// Represents the result of validating the contents using
+/// \c validateOnDiskUnifiedCASDatabasesIfNeeded.
+///
+/// Note: invalid results are handled as an \c Error.
+enum class ValidationResult {
+  /// The data is already valid.
+  Valid,
+  /// The data was invalid, but was recovered.
+  RecoveredValid,
+  /// Validation was skipped, as it was not needed.
+  Skipped,
+};
+
+/// Validate the data in \p Path, if needed to ensure correctness.
+///
+/// \param Path directory for the on-disk database.
+/// \param CheckHash Whether to validate hashes match the data.
+/// \param AllowRecovery Whether to automatically recover from invalid data by
+/// marking the files for garbage collection.
+/// \param ForceValidation Whether to force validation to occur even if it
+/// should not be necessary.
+/// \param LLVMCasBinary If provided, validation is performed out-of-process
+/// using the given \c llvm-cas executable which protects against crashes
+/// during validation. Otherwise validation is performed in-process.
+///
+/// \returns \c Valid if the data is already valid, \c RecoveredValid if data
+/// was invalid but has been cleared, \c Skipped if validation is not needed,
+/// or an \c Error if validation cannot be performed or if the data is left
+/// in an invalid state because \p AllowRecovery is false.
+Expected<ValidationResult> validateOnDiskUnifiedCASDatabasesIfNeeded(
+    StringRef Path, bool CheckHash, bool AllowRecovery, bool ForceValidation,
+    std::optional<StringRef> LLVMCasBinary);
+
 } // namespace llvm::cas
 
 #endif // LLVM_CAS_BUILTINUNIFIEDCASDATABASES_H

llvm/include/llvm/CAS/OnDiskCASLogger.h

@@ -62,6 +62,11 @@ class OnDiskCASLogger {
   void log_MappedFileRegionBumpPtr_allocate(void *Region, TrieOffset Off,
                                             size_t Size);
   void log_UnifiedOnDiskCache_collectGarbage(StringRef Path);
+  void log_UnifiedOnDiskCache_validateIfNeeded(
+      StringRef Path, uint64_t BootTime, uint64_t ValidationTime,
+      bool CheckHash, bool AllowRecovery, bool Force,
+      std::optional<StringRef> LLVMCas, StringRef ValidationError, bool Skipped,
+      bool Recovered);
   void log_TempFile_create(StringRef Name);
   void log_TempFile_keep(StringRef TmpName, StringRef Name, std::error_code EC);
   void log_TempFile_remove(StringRef TmpName, std::error_code EC);

llvm/include/llvm/CAS/UnifiedOnDiskCache.h

@@ -9,6 +9,7 @@
 #ifndef LLVM_CAS_UNIFIEDONDISKCACHE_H
 #define LLVM_CAS_UNIFIEDONDISKCACHE_H
 
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
 #include "llvm/CAS/OnDiskGraphDB.h"
 
 namespace llvm::cas::ondisk {
@@ -82,6 +83,34 @@ class UnifiedOnDiskCache {
        OnDiskGraphDB::FaultInPolicy FaultInPolicy =
            OnDiskGraphDB::FaultInPolicy::FullTree);
 
+  /// Validate the data in \p Path, if needed to ensure correctness.
+  ///
+  /// Note: if invalid data is detected and \p AllowRecovery is true, then
+  /// recovery requires exclusive access to the CAS and it is an error to
+  /// attempt recovery if there is concurrent use of the CAS.
+  ///
+  /// \param Path directory for the on-disk database.
+  /// \param HashName Identifier name for the hashing algorithm that is going to
+  /// be used.
+  /// \param HashByteSize Size for the object digest hash bytes.
+  /// \param CheckHash Whether to validate hashes match the data.
+  /// \param AllowRecovery Whether to automatically recover from invalid data by
+  /// marking the files for garbage collection.
+  /// \param ForceValidation Whether to force validation to occur even if it
+  /// should not be necessary.
+  /// \param LLVMCasBinary If provided, validation is performed out-of-process
+  /// using the given \c llvm-cas executable which protects against crashes
+  /// during validation. Otherwise validation is performed in-process.
+  ///
+  /// \returns \c Valid if the data is already valid, \c RecoveredValid if data
+  /// was invalid but has been cleared, \c Skipped if validation is not needed,
+  /// or an \c Error if validation cannot be performed or if the data is left
+  /// in an invalid state because \p AllowRecovery is false.
+  static Expected<ValidationResult>
+  validateIfNeeded(StringRef Path, StringRef HashName, unsigned HashByteSize,
+                   bool CheckHash, bool AllowRecovery, bool ForceValidation,
+                   std::optional<StringRef> LLVMCasBinary);
+
   /// This is called implicitly at destruction time, so it is not required for a
   /// client to call this. After calling \p close the only method that is valid
   /// to call is \p needsGarbageCollection.

llvm/lib/CAS/BuiltinUnifiedCASDatabases.cpp

@@ -23,3 +23,16 @@ cas::createOnDiskUnifiedCASDatabases(StringRef Path) {
   auto AC = builtin::createActionCacheFromUnifiedOnDiskCache(std::move(UniDB));
   return std::make_pair(std::move(CAS), std::move(AC));
 }
+
+Expected<ValidationResult> cas::validateOnDiskUnifiedCASDatabasesIfNeeded(
+    StringRef Path, bool CheckHash, bool AllowRecovery, bool ForceValidation,
+    std::optional<StringRef> LLVMCasBinary) {
+#if LLVM_ENABLE_ONDISK_CAS
+  return ondisk::UnifiedOnDiskCache::validateIfNeeded(
+      Path, builtin::BuiltinCASContext::getHashName(),
+      sizeof(builtin::HashType), CheckHash, AllowRecovery, ForceValidation,
+      LLVMCasBinary);
+#else
+  return createStringError(inconvertibleErrorCode(), "OnDiskCache is disabled");
+#endif
+}

llvm/lib/CAS/OnDiskCASLogger.cpp

@@ -211,6 +211,25 @@ void OnDiskCASLogger::log_UnifiedOnDiskCache_collectGarbage(StringRef Path) {
   Log << "collect garbage '" << Path << "'";
 }
 
+void OnDiskCASLogger::log_UnifiedOnDiskCache_validateIfNeeded(
+    StringRef Path, uint64_t BootTime, uint64_t ValidationTime, bool CheckHash,
+    bool AllowRecovery, bool Force, std::optional<StringRef> LLVMCas,
+    StringRef ValidationError, bool Skipped, bool Recovered) {
+  TextLogLine Log(OS);
+  Log << "validate-if-needed '" << Path << "'";
+  Log << " boot=" << BootTime << " last-valid=" << ValidationTime;
+  Log << " check-hash=" << CheckHash << " allow-recovery=" << AllowRecovery;
+  Log << " force=" << Force;
+  if (LLVMCas)
+    Log << " llvm-cas=" << *LLVMCas;
+  if (Skipped)
+    Log << " skipped";
+  if (Recovered)
+    Log << " recovered";
+  if (!ValidationError.empty())
+    Log << " data was invalid " << ValidationError;
+}
+
 void OnDiskCASLogger::log_TempFile_create(StringRef Name) {
   TextLogLine Log(OS);
   Log << "standalone file create '" << Name << "'";