Teach swift to compute cache key for compiler outputs

cachemeifyoucan · cachemeifyoucan · commit c7d66b884528 · 2023-04-24T13:55:38.000-07:00
Teach swift how to serialize its input into CAS to create a cache key
for compiler outputs. To compute the cache key for the output, it first
needs to compute a base-key for the compiler invocation. The base key is
computed from: swift compiler version and the command-line arguments for
the invocation.

Each compiler output from swift will gets its own key. The key for the
output is computed from: the base key for the compiler invocation + the
primary input for the output + the output type.
diff --git a/include/swift/Basic/FileTypes.def b/include/swift/Basic/FileTypes.def
@@ -98,5 +98,8 @@ TYPE("pch",                 PCH,                       "pch",             "")
 TYPE("none",                Nothing,                   "",                "")
 
 TYPE("abi-baseline-json",   SwiftABIDescriptor,        "abi.json",        "")
+TYPE("fixit",               SwiftFixIt,                "",                "")
+TYPE("module-semantic-info", ModuleSemanticInfo,       "",                "")
+TYPE("cached-diagnostics",  CachedDiagnostics,         "",                "")
 
 #undef TYPE
diff --git a/include/swift/Basic/SupplementaryOutputPaths.h b/include/swift/Basic/SupplementaryOutputPaths.h
@@ -13,6 +13,7 @@
 #ifndef SWIFT_FRONTEND_SUPPLEMENTARYOUTPUTPATHS_H
 #define SWIFT_FRONTEND_SUPPLEMENTARYOUTPUTPATHS_H
 
+#include "swift/Basic/FileTypes.h"
 #include "swift/Basic/LLVM.h"
 #include "llvm/IR/Function.h"
 
@@ -201,6 +202,48 @@ struct SupplementaryOutputPaths {
       fn(ModuleSemanticInfoOutputPath);
   }
 
+  void forEachSetOutputAndType(
+      llvm::function_ref<void(const std::string &, file_types::ID)> fn) const {
+    if (!ClangHeaderOutputPath.empty())
+      fn(ClangHeaderOutputPath, file_types::ID::TY_ClangHeader);
+    if (!ModuleOutputPath.empty())
+      fn(ModuleOutputPath, file_types::ID::TY_SwiftModuleFile);
+    if (!ModuleSourceInfoOutputPath.empty())
+      fn(ModuleSourceInfoOutputPath, file_types::ID::TY_SwiftSourceInfoFile);
+    if (!ModuleDocOutputPath.empty())
+      fn(ModuleDocOutputPath, file_types::ID::TY_SwiftModuleDocFile);
+    if (!DependenciesFilePath.empty())
+      fn(DependenciesFilePath, file_types::ID::TY_Dependencies);
+    if (!ReferenceDependenciesFilePath.empty())
+      fn(ReferenceDependenciesFilePath, file_types::ID::TY_SwiftDeps);
+    if (!SerializedDiagnosticsPath.empty())
+      fn(SerializedDiagnosticsPath, file_types::ID::TY_SerializedDiagnostics);
+    if (!FixItsOutputPath.empty())
+      fn(FixItsOutputPath, file_types::ID::TY_SwiftFixIt);
+    if (!LoadedModuleTracePath.empty())
+      fn(LoadedModuleTracePath, file_types::ID::TY_ModuleTrace);
+    if (!TBDPath.empty())
+      fn(TBDPath, file_types::ID::TY_TBD);
+    if (!ModuleInterfaceOutputPath.empty())
+      fn(ModuleInterfaceOutputPath,
+         file_types::ID::TY_SwiftModuleInterfaceFile);
+    if (!PrivateModuleInterfaceOutputPath.empty())
+      fn(PrivateModuleInterfaceOutputPath,
+         file_types::ID::TY_PrivateSwiftModuleInterfaceFile);
+    if (!ModuleSummaryOutputPath.empty())
+      fn(ModuleSummaryOutputPath, file_types::ID::TY_SwiftModuleSummaryFile);
+    if (!ABIDescriptorOutputPath.empty())
+      fn(ABIDescriptorOutputPath, file_types::ID::TY_SwiftABIDescriptor);
+    if (!ConstValuesOutputPath.empty())
+      fn(ConstValuesOutputPath, file_types::ID::TY_ConstValues);
+    if (!YAMLOptRecordPath.empty())
+      fn(YAMLOptRecordPath, file_types::ID::TY_YAMLOptRecord);
+    if (!BitstreamOptRecordPath.empty())
+      fn(BitstreamOptRecordPath, file_types::ID::TY_BitstreamOptRecord);
+    if (!ModuleSemanticInfoOutputPath.empty())
+      fn(ModuleSemanticInfoOutputPath, file_types::ID::TY_ModuleSemanticInfo);
+  }
+
   bool empty() const {
     return ClangHeaderOutputPath.empty() && ModuleOutputPath.empty() &&
            ModuleDocOutputPath.empty() && DependenciesFilePath.empty() &&
diff --git a/include/swift/Frontend/CompileJobCacheKey.h b/include/swift/Frontend/CompileJobCacheKey.h
@@ -0,0 +1,48 @@
+//===--- CompileJobCacheKey.h - compile cache key methods -------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains declarations of utility methods for creating cache keys
+// for compilation jobs.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_COMPILEJOBCACHEKEY_H
+#define SWIFT_COMPILEJOBCACHEKEY_H
+
+#include "swift/AST/DiagnosticEngine.h"
+#include "swift/Basic/FileTypes.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/Error.h"
+
+namespace swift {
+
+/// Compute CompileJobBaseKey from swift-frontend command-line arguments.
+/// CompileJobBaseKey represents the core inputs and arguments, and is used as a
+/// base to compute keys for each compiler outputs.
+// TODO: switch to create key from CompilerInvocation after we can canonicalize
+// arguments.
+llvm::Expected<llvm::cas::ObjectRef>
+createCompileJobBaseCacheKey(llvm::cas::ObjectStore &CAS,
+                             ArrayRef<const char *> Args);
+
+/// Compute CompileJobKey for the compiler outputs. The key for the output
+/// is computed from the base key for the compilation, the output kind and the
+/// input file path that is associated with this specific output.
+llvm::Expected<llvm::cas::ObjectRef>
+createCompileJobCacheKeyForOutput(llvm::cas::ObjectStore &CAS,
+                                  llvm::cas::ObjectRef BaseKey,
+                                  StringRef ProducingInput,
+                                  file_types::ID OutputType);
+}
+
+#endif
diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h
@@ -455,6 +455,7 @@ class CompilerInstance {
   /// the file buffer provided by CAS needs to outlive the SourceMgr.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> ResultCache;
+  Optional<llvm::cas::ObjectRef> CompileJobBaseKey;
 
   SourceManager SourceMgr;
   DiagnosticEngine Diagnostics{SourceMgr};
@@ -544,6 +545,9 @@ class CompilerInstance {
   std::shared_ptr<llvm::cas::ObjectStore> getSharedCASInstance() const {
     return CAS;
   }
+  Optional<llvm::cas::ObjectRef> getCompilerBaseKey() const {
+    return CompileJobBaseKey;
+  }
 
   ASTContext &getASTContext() { return *Context; }
   const ASTContext &getASTContext() const { return *Context; }
@@ -646,7 +650,8 @@ class CompilerInstance {
   }
 
   /// Returns true if there was an error during setup.
-  bool setup(const CompilerInvocation &Invocation, std::string &Error);
+  bool setup(const CompilerInvocation &Invocation, std::string &Error,
+             ArrayRef<const char *> Args = {});
 
   const CompilerInvocation &getInvocation() const { return Invocation; }
 
@@ -667,7 +672,7 @@ class CompilerInstance {
   bool setUpASTContextIfNeeded();
   void setupStatsReporter();
   void setupDependencyTrackerIfNeeded();
-  bool setupCASIfNeeded();
+  bool setupCASIfNeeded(ArrayRef<const char *> Args);
   void setupOutputBackend();
 
   /// \return false if successful, true on error.
diff --git a/include/swift/Frontend/FrontendInputsAndOutputs.h b/include/swift/Frontend/FrontendInputsAndOutputs.h
@@ -39,6 +39,10 @@ class FrontendInputsAndOutputs {
   llvm::StringMap<unsigned> PrimaryInputsByName;
   std::vector<unsigned> PrimaryInputsInOrder;
 
+  /// The file type for main output files. Assuming all inputs produce the
+  /// same kind of output.
+  file_types::ID PrincipalOutputType = file_types::ID::TY_INVALID;
+
   /// In Single-threaded WMO mode, all inputs are used
   /// both for importing and compiling.
   bool IsSingleThreadedWMO = false;
@@ -190,13 +194,17 @@ class FrontendInputsAndOutputs {
       ArrayRef<std::string> outputFiles,
       ArrayRef<SupplementaryOutputPaths> supplementaryOutputs,
       ArrayRef<std::string> outputFilesForIndexUnits = None);
+  void setPrincipalOutputType(file_types::ID type) {
+    PrincipalOutputType = type;
+  }
 
 public:
   unsigned countOfInputsProducingMainOutputs() const;
 
   bool hasInputsProducingMainOutputs() const {
     return countOfInputsProducingMainOutputs() != 0;
   }
+  file_types::ID getPrincipalOutputType() const { return PrincipalOutputType; }
 
   const InputFile &firstInputProducingOutput() const;
   const InputFile &lastInputProducingOutput() const;
diff --git a/lib/Basic/FileTypes.cpp b/lib/Basic/FileTypes.cpp
@@ -109,6 +109,9 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_IndexData:
   case file_types::TY_BitstreamOptRecord:
   case file_types::TY_IndexUnitOutputPath:
+  case file_types::TY_SwiftFixIt:
+  case file_types::TY_ModuleSemanticInfo:
+  case file_types::TY_CachedDiagnostics:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");
@@ -162,6 +165,9 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_IndexUnitOutputPath:
   case file_types::TY_SwiftABIDescriptor:
   case file_types::TY_ConstValues:
+  case file_types::TY_SwiftFixIt:
+  case file_types::TY_ModuleSemanticInfo:
+  case file_types::TY_CachedDiagnostics:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");
@@ -215,6 +221,9 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
   case file_types::TY_IndexUnitOutputPath:
   case file_types::TY_SwiftABIDescriptor:
   case file_types::TY_ConstValues:
+  case file_types::TY_SwiftFixIt:
+  case file_types::TY_ModuleSemanticInfo:
+  case file_types::TY_CachedDiagnostics:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
@@ -2084,6 +2084,9 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       case file_types::TY_JSONFeatures:
       case file_types::TY_SwiftABIDescriptor:
       case file_types::TY_ConstValues:
+      case file_types::TY_SwiftFixIt:
+      case file_types::TY_ModuleSemanticInfo:
+      case file_types::TY_CachedDiagnostics:
         // We could in theory handle assembly or LLVM input, but let's not.
         // FIXME: What about LTO?
         Diags.diagnose(SourceLoc(), diag::error_unexpected_input_file,
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
@@ -753,6 +753,9 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
   case file_types::TY_IndexUnitOutputPath:
   case file_types::TY_SwiftABIDescriptor:
   case file_types::TY_ConstValues:
+  case file_types::TY_SwiftFixIt:
+  case file_types::TY_ModuleSemanticInfo:
+  case file_types::TY_CachedDiagnostics:
     llvm_unreachable("Output type can never be primary output.");
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID");
@@ -1014,6 +1017,9 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case file_types::TY_IndexUnitOutputPath:
     case file_types::TY_SwiftABIDescriptor:
     case file_types::TY_ConstValues:
+    case file_types::TY_SwiftFixIt:
+    case file_types::TY_ModuleSemanticInfo:
+    case file_types::TY_CachedDiagnostics:
       llvm_unreachable("Output type can never be primary output.");
     case file_types::TY_INVALID:
       llvm_unreachable("Invalid type ID");
diff --git a/lib/Frontend/CMakeLists.txt b/lib/Frontend/CMakeLists.txt
@@ -3,6 +3,7 @@ add_swift_host_library(swiftFrontend STATIC
   ArgsToFrontendInputsConverter.cpp
   ArgsToFrontendOptionsConverter.cpp
   ArgsToFrontendOutputsConverter.cpp
+  CompileJobCacheKey.cpp
   CompilerInvocation.cpp
   DependencyVerifier.cpp
   DiagnosticVerifier.cpp
diff --git a/lib/Frontend/CompileJobCacheKey.cpp b/lib/Frontend/CompileJobCacheKey.cpp
@@ -0,0 +1,83 @@
+//===--- CompileJobCacheKey.cpp - compile cache key methods ---------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains utility methods for creating compile job cache keys.
+//
+//===----------------------------------------------------------------------===//
+
+#include <swift/Frontend/CompileJobCacheKey.h>
+#include <swift/Basic/Version.h>
+#include <llvm/ADT/SmallString.h>
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/CAS/HierarchicalTreeBuilder.h"
+#include "llvm/CAS/ObjectStore.h"
+
+using namespace swift;
+
+// TODO: Rewrite this into CASNodeSchema.
+llvm::Expected<llvm::cas::ObjectRef> swift::createCompileJobBaseCacheKey(
+    llvm::cas::ObjectStore &CAS, ArrayRef<const char *> Args) {
+  SmallString<256> CommandLine;
+
+  // TODO: Improve this list.
+  static const std::vector<std::string> removeArgAndNext = {
+      "-o", "-supplementary-output-file-map", "-serialize-diagnostics-path",
+      "-num-threads", "-cas-path"};
+
+  // Don't count the `-frontend` in the first location since only frontend
+  // invocation can have a cache key.
+  if (Args.size() > 1 && StringRef(Args.front()) == "-frontend")
+    Args = Args.drop_front();
+
+  bool SkipNext = false;
+  for (StringRef Arg : Args) {
+    if (SkipNext) {
+      SkipNext = false;
+      continue;
+    }
+    if (llvm::is_contained(removeArgAndNext, Arg)) {
+      SkipNext = true;
+      continue;
+    }
+    CommandLine.append(Arg);
+    CommandLine.push_back(0);
+  }
+
+  llvm::cas::HierarchicalTreeBuilder Builder;
+  auto CMD = CAS.storeFromString(None, CommandLine);
+  if (!CMD)
+    return CMD.takeError();
+  Builder.push(*CMD, llvm::cas::TreeEntry::Regular, "command-line");
+
+  // FIXME: The version is maybe insufficient...
+  auto Version = CAS.storeFromString(None, version::getSwiftFullVersion());
+  if (!Version)
+    return Version.takeError();
+  Builder.push(*Version, llvm::cas::TreeEntry::Regular, "version");
+
+  if (auto Out = Builder.create(CAS))
+    return Out->getRef();
+  else
+    return Out.takeError();
+}
+
+llvm::Expected<llvm::cas::ObjectRef> swift::createCompileJobCacheKeyForOutput(
+    llvm::cas::ObjectStore &CAS, llvm::cas::ObjectRef BaseKey,
+    StringRef ProducingInput, file_types::ID OutputType) {
+  SmallString<256> OutputInfo;
+
+  // Encode the OutputType as first byte, then append the input file path.
+  OutputInfo.emplace_back((char)OutputType);
+  OutputInfo.append(ProducingInput);
+
+  return CAS.storeFromString({BaseKey}, OutputInfo);
+}
diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp
@@ -26,6 +26,7 @@
 #include "swift/Basic/FileTypes.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/Basic/Statistic.h"
+#include "swift/Frontend/CompileJobCacheKey.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/SIL/SILModule.h"
@@ -398,7 +399,7 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() {
     DepTracker->addDependency(path, /*isSystem=*/false);
 }
 
-bool CompilerInstance::setupCASIfNeeded() {
+bool CompilerInstance::setupCASIfNeeded(ArrayRef<const char *> Args) {
   const auto &Opts = getInvocation().getFrontendOptions();
   if (!Opts.EnableCAS)
     return false;
@@ -429,6 +430,13 @@ bool CompilerInstance::setupCASIfNeeded() {
     }
   }
 
+  auto BaseKey = createCompileJobBaseCacheKey(*CAS, Args);
+  if (!BaseKey) {
+    Diagnostics.diagnose(SourceLoc(), diag::error_cas,
+                         toString(BaseKey.takeError()));
+    return true;
+  }
+  CompileJobBaseKey = *BaseKey;
   return false;
 }
 
@@ -452,10 +460,10 @@ void CompilerInstance::setupOutputBackend() {
 }
 
 bool CompilerInstance::setup(const CompilerInvocation &Invoke,
-                             std::string &Error) {
+                             std::string &Error, ArrayRef<const char *> Args) {
   Invocation = Invoke;
 
-  if (setupCASIfNeeded()) {
+  if (setupCASIfNeeded(Args)) {
     Error = "Setting up CAS failed";
     return true;
   }
diff --git a/lib/Frontend/FrontendInputsAndOutputs.cpp b/lib/Frontend/FrontendInputsAndOutputs.cpp
@@ -39,6 +39,7 @@ FrontendInputsAndOutputs::FrontendInputsAndOutputs(
     addInput(input);
   IsSingleThreadedWMO = other.IsSingleThreadedWMO;
   ShouldRecoverMissingInputs = other.ShouldRecoverMissingInputs;
+  OutputType = other.OutputType;
 }
 
 FrontendInputsAndOutputs &FrontendInputsAndOutputs::
@@ -48,6 +49,7 @@ operator=(const FrontendInputsAndOutputs &other) {
     addInput(input);
   IsSingleThreadedWMO = other.IsSingleThreadedWMO;
   ShouldRecoverMissingInputs = other.ShouldRecoverMissingInputs;
+  OutputType = other.OutputType;
   return *this;
 }
 
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp

Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,7 @@ FrontendInputsAndOutputs::FrontendInputsAndOutputs(`
`39`	`39`	`addInput(input);`
`40`	`40`	`IsSingleThreadedWMO = other.IsSingleThreadedWMO;`
`41`	`41`	`ShouldRecoverMissingInputs = other.ShouldRecoverMissingInputs;`
	`42`	`+ OutputType = other.OutputType;`
`42`	`43`	`}`
`43`	`44`
`44`	`45`	`FrontendInputsAndOutputs &FrontendInputsAndOutputs::`
`@@ -48,6 +49,7 @@ operator=(const FrontendInputsAndOutputs &other) {`
`48`	`49`	`addInput(input);`
`49`	`50`	`IsSingleThreadedWMO = other.IsSingleThreadedWMO;`
`50`	`51`	`ShouldRecoverMissingInputs = other.ShouldRecoverMissingInputs;`
	`52`	`+ OutputType = other.OutputType;`
`51`	`53`	`return *this;`
`52`	`54`	`}`
`53`	`55`