Merge pull request #65058 from cachemeifyoucan/eng/PR-swift-cas-fs

[CAS] Integrate CAS into swift compiler

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -485,6 +485,11 @@ ERROR(error_output_missing,none,
 REMARK(matching_output_produced,none,
       "produced matching output file '%0' for deterministic check: hash '%1'", (StringRef, StringRef))
 
+// CAS related diagnostics
+ERROR(error_create_cas, none, "failed to create CAS '%0' (%1)", (StringRef, StringRef))
+ERROR(error_invalid_cas_id, none, "invalid CASID '%0' (%1)", (StringRef, StringRef))
+ERROR(error_cas, none, "CAS error encountered: %0", (StringRef))
+
 // Dependency Verifier Diagnostics
 ERROR(missing_member_dependency,none,
       "expected "

include/swift/Basic/LangOptions.h

@@ -804,6 +804,9 @@ namespace swift {
     /// import, but it can affect Clang's IR generation of static functions.
     std::string Optimization;
 
+    /// clang CASOptions.
+    std::string CASPath;
+
     /// Disable validating the persistent PCH.
     bool PCHDisableValidation = false;
 

include/swift/Frontend/Frontend.h

@@ -44,6 +44,8 @@
 #include "clang/Basic/FileManager.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/BLAKE3.h"
 #include "llvm/Support/HashingOutputBackend.h"
@@ -447,6 +449,13 @@ class CompilerInvocation {
 /// times on a single CompilerInstance is not permitted.
 class CompilerInstance {
   CompilerInvocation Invocation;
+
+  /// CAS Instances.
+  /// This needs to be declared before SourceMgr because when using CASFS,
+  /// the file buffer provided by CAS needs to outlive the SourceMgr.
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  std::shared_ptr<llvm::cas::ActionCache> ResultCache;
+
   SourceManager SourceMgr;
   DiagnosticEngine Diagnostics{SourceMgr};
   std::unique_ptr<ASTContext> Context;
@@ -516,9 +525,6 @@ class CompilerInstance {
   llvm::vfs::FileSystem &getFileSystem() const {
     return *SourceMgr.getFileSystem();
   }
-  void setFileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
-    SourceMgr.setFileSystem(FS);
-  }
 
   llvm::vfs::OutputBackend &getOutputBackend() const {
     return *OutputBackend;
@@ -530,6 +536,15 @@ class CompilerInstance {
   using HashingBackendPtrTy = llvm::IntrusiveRefCntPtr<HashBackendTy>;
   HashingBackendPtrTy getHashingBackend() { return HashBackend; }
 
+  llvm::cas::ObjectStore &getObjectStore() const { return *CAS; }
+  llvm::cas::ActionCache &getActionCache() const { return *ResultCache; }
+  std::shared_ptr<llvm::cas::ActionCache> getSharedCacheInstance() const {
+    return ResultCache;
+  }
+  std::shared_ptr<llvm::cas::ObjectStore> getSharedCASInstance() const {
+    return CAS;
+  }
+
   ASTContext &getASTContext() { return *Context; }
   const ASTContext &getASTContext() const { return *Context; }
 
@@ -652,6 +667,7 @@ class CompilerInstance {
   bool setUpASTContextIfNeeded();
   void setupStatsReporter();
   void setupDependencyTrackerIfNeeded();
+  bool setupCASIfNeeded();
   void setupOutputBackend();
 
   /// \return false if successful, true on error.

include/swift/Frontend/FrontendOptions.h

@@ -122,6 +122,15 @@ class FrontendOptions {
   /// The module for which we should verify all of the generic signatures.
   std::string VerifyGenericSignaturesInModule;
 
+  /// Use CAS.
+  bool EnableCAS = false;
+
+  /// The CAS Path.
+  std::string CASPath;
+
+  /// CASFS Root.
+  std::string CASFSRootID;
+
   /// Number of retry opening an input file if the previous opening returns
   /// bad file descriptor error.
   unsigned BadFileDescriptorRetryCount = 0;

include/swift/Option/Options.td

@@ -1810,8 +1810,20 @@ def gcc_toolchain: Separate<["-"], "gcc-toolchain">,
   MetaVarName<"<path>">,
   HelpText<"Specify a directory where the clang importer and clang linker can find headers and libraries">;
 
+def cas_path: Separate<["-"], "cas-path">,
+  Flags<[FrontendOption, NewDriverOnlyOption]>,
+  HelpText<"Path to CAS">, MetaVarName<"<path>">;
+
 // END ONLY SUPPORTED IN NEW DRIVER
 
+def enable_cas: Flag<["-"], "enable-cas">,
+  Flags<[FrontendOption, NoDriverOption]>,
+  HelpText<"Enable CAS for swift-frontend">;
+
+def cas_fs: Separate<["-"], "cas-fs">,
+  Flags<[FrontendOption, NoDriverOption]>,
+  HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"<cas-id>">;
+
 def load_plugin_library:
   Separate<["-"], "load-plugin-library">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -23,6 +23,7 @@
 #include "swift/Strings.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
@@ -349,6 +350,12 @@ bool ArgsToFrontendOptionsConverter::convert(
   for (auto A : Args.getAllArgValues(options::OPT_block_list_file)) {
     Opts.BlocklistConfigFilePaths.push_back(A);
   }
+
+  Opts.EnableCAS = Args.hasArg(OPT_enable_cas);
+  Opts.CASPath =
+      Args.getLastArgValue(OPT_cas_path, llvm::cas::getDefaultOnDiskCASPath());
+  Opts.CASFSRootID = Args.getLastArgValue(OPT_cas_fs);
+
   return false;
 }
 

lib/Frontend/CompilerInvocation.cpp

@@ -1465,6 +1465,11 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts,
   Opts.DisableSourceImport |=
       Args.hasArg(OPT_disable_clangimporter_source_import);
 
+  // Forward the FrontendOptions to clang importer option so it can be
+  // accessed when creating clang module compilation invocation.
+  if (FrontendOpts.EnableCAS)
+    Opts.CASPath = FrontendOpts.CASPath;
+
   return false;
 }
 

lib/Frontend/Frontend.cpp

@@ -41,6 +41,9 @@
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
+#include "llvm/CAS/CASFileSystem.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -395,6 +398,40 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() {
     DepTracker->addDependency(path, /*isSystem=*/false);
 }
 
+bool CompilerInstance::setupCASIfNeeded() {
+  const auto &Opts = getInvocation().getFrontendOptions();
+  if (!Opts.EnableCAS)
+    return false;
+
+  auto MaybeCache = llvm::cas::createOnDiskUnifiedCASDatabases(Opts.CASPath);
+  if (!MaybeCache) {
+    Diagnostics.diagnose(SourceLoc(), diag::error_create_cas, Opts.CASPath,
+                         toString(MaybeCache.takeError()));
+    return true;
+  }
+  CAS = std::move(MaybeCache->first);
+  ResultCache = std::move(MaybeCache->second);
+
+  // create baseline key.
+  llvm::Optional<llvm::cas::ObjectRef> FSRef;
+  if (!Opts.CASFSRootID.empty()) {
+    auto CASFSID = CAS->parseID(Opts.CASFSRootID);
+    if (!CASFSID) {
+      Diagnostics.diagnose(SourceLoc(), diag::error_cas,
+                           toString(CASFSID.takeError()));
+      return true;
+    }
+    FSRef = CAS->getReference(*CASFSID);
+    if (!FSRef) {
+      Diagnostics.diagnose(SourceLoc(), diag::error_cas,
+                           "-cas-fs value does not exist in CAS");
+      return true;
+    }
+  }
+
+  return false;
+}
+
 void CompilerInstance::setupOutputBackend() {
   // Skip if output backend is not setup, default to OnDiskOutputBackend.
   if (OutputBackend)
@@ -418,6 +455,11 @@ bool CompilerInstance::setup(const CompilerInvocation &Invoke,
                              std::string &Error) {
   Invocation = Invoke;
 
+  if (setupCASIfNeeded()) {
+    Error = "Setting up CAS failed";
+    return true;
+  }
+
   setupDependencyTrackerIfNeeded();
   setupOutputBackend();
 
@@ -463,6 +505,26 @@ bool CompilerInstance::setup(const CompilerInvocation &Invoke,
 }
 
 bool CompilerInstance::setUpVirtualFileSystemOverlays() {
+  if (Invocation.getFrontendOptions().EnableCAS &&
+      !Invocation.getFrontendOptions().CASFSRootID.empty()) {
+    // Set up CASFS as BaseFS.
+    auto RootID = CAS->parseID(Invocation.getFrontendOptions().CASFSRootID);
+    if (!RootID) {
+      Diagnostics.diagnose(SourceLoc(), diag::error_invalid_cas_id,
+                           Invocation.getFrontendOptions().CASFSRootID,
+                           toString(RootID.takeError()));
+      return true;
+    }
+    auto FS = llvm::cas::createCASFileSystem(*CAS, *RootID);
+    if (!FS) {
+      Diagnostics.diagnose(SourceLoc(), diag::error_invalid_cas_id,
+                           Invocation.getFrontendOptions().CASFSRootID,
+                           toString(FS.takeError()));
+      return true;
+    }
+    SourceMgr.setFileSystem(std::move(*FS));
+  }
+
   auto ExpectedOverlay =
       Invocation.getSearchPathOptions().makeOverlayFileSystem(
           SourceMgr.getFileSystem());

test/CAS/Inputs/objc.h

@@ -0,0 +1 @@
+int test(int);

test/CAS/cas_fs.swift

@@ -0,0 +1,29 @@
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/empty
+// RUN: mkdir -p %t/cas
+
+// RUN: llvm-cas --cas %t/cas --ingest --data %t/empty > %t/empty.casid
+// RUN: not %target-swift-frontend -typecheck -enable-cas -cas-fs @%t/empty.casid -cas-path %t/cas %s 2>&1 | %FileCheck %s --check-prefix NO-INPUTS
+// NO-INPUTS: error: error opening input file
+
+// RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/source.casid
+// RUN: not %target-swift-frontend -typecheck -enable-cas -cas-fs @%t/source.casid -cas-path %t/cas %s 2>&1 | %FileCheck %s --check-prefix NO-RESOURCES
+// NO-RESOURCES: error: unable to set working directory
+// NO-RESOURCES: error: unable to load standard library
+
+/// Ingest the resource directory to satisfy the file system requirement. Also switch CWD to resource dir.
+// RUN: llvm-cas --cas %t/cas --merge @%t/source.casid %test-resource-dir > %t/full.casid
+// RUN: cd %test-resource-dir
+// RUN: %target-swift-frontend -typecheck -enable-cas -cas-fs @%t/full.casid -cas-path %t/cas %s
+
+/// Try clang importer.
+// RUN: not %target-swift-frontend -typecheck -enable-cas -cas-fs @%t/full.casid -cas-path %t/cas %s -import-objc-header %S/Inputs/objc.h 2>&1 | %FileCheck %s --check-prefix NO-BRIDGING-HEADER
+// NO-BRIDGING-HEADER: error: bridging header
+
+// RUN: llvm-cas --cas %t/cas --merge @%t/full.casid %S/Inputs/objc.h > %t/bridging_header.casid
+// RUN: %target-swift-frontend -typecheck -enable-cas -cas-fs @%t/bridging_header.casid -cas-path %t/cas %s -import-objc-header %S/Inputs/objc.h
+
+/// Clean the CAS to save space.
+// RUN: %empty-directory(%t)
+
+func testFunc() {}

test/CAS/lit.local.cfg

@@ -0,0 +1,4 @@
+# FIXME: CAS file system doesn't support windows. Unsupported for now.
+import platform
+if platform.system() == 'Windows':
+    config.unsupported = True

test/CMakeLists.txt

@@ -85,6 +85,7 @@ function(get_test_dependencies SDK result_var_name)
       llvm-ar
       llvm-as
       llvm-bcanalyzer
+      llvm-cas
       llvm-cov
       llvm-dis
       llvm-dwarfdump