Merge pull request #34196 from CodaFi/crossbeam

Turn On Cross-Module Incremental Builds!

Author: CodaFi

include/swift/AST/FineGrainedDependencyFormat.h

@@ -135,8 +135,32 @@ bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags,
                                            llvm::StringRef path,
                                            const SourceFileDepGraph &g);
 
+/// Enumerates the supported set of purposes for writing out or reading in
+/// swift dependency information into a file. These can be used to influence
+/// the structure of the resulting data that is produced by the serialization
+/// machinery defined here.
+enum class Purpose : bool {
+  /// Write out fine grained dependency metadata suitable for embedding in
+  /// \c .swiftmodule file.
+  ///
+  /// The resulting metadata does not contain the usual block descriptor header
+  /// nor does it contain a leading magic signature, which would otherwise
+  /// disrupt clients and tools that do not expect them to be present such as
+  /// llvm-bcanalyzer.
+  ForSwiftModule = false,
+  /// Write out fine grained dependency metadata suitable for a standalone
+  /// \c .swiftdeps file.
+  ///
+  /// The resulting metadata will contain a leading magic signature and block
+  /// descriptor header.
+  ForSwiftDeps = true,
+};
+
+/// Tries to write out the given dependency graph with the given
+/// bitstream writer.
 void writeFineGrainedDependencyGraph(llvm::BitstreamWriter &Out,
-                                     const SourceFileDepGraph &g);
+                                     const SourceFileDepGraph &g,
+                                     Purpose purpose);
 
 } // namespace fine_grained_dependencies
 } // namespace swift

include/swift/Driver/FineGrainedDependencyDriverGraph.h

@@ -278,8 +278,6 @@ class ModuleDepGraph {
     assert(swiftDeps.hasValue() && "Don't call me for expats.");
     auto iter = jobsBySwiftDeps.find(swiftDeps.getValue());
     assert(iter != jobsBySwiftDeps.end() && "All jobs should be tracked.");
-    assert(getSwiftDeps(iter->second) == swiftDeps.getValue() &&
-           "jobsBySwiftDeps should be inverse of getSwiftDeps.");
     return iter->second;
   }
 

include/swift/Option/Options.td

@@ -573,6 +573,7 @@ def experimental_cxx_stdlib :
 
 def enable_experimental_cross_module_incremental_build :
   Flag<["-"], "enable-experimental-cross-module-incremental-build">,
+  Flags<[FrontendOption]>,
   HelpText<"(experimental) Enable cross-module incremental build metadata and "
            "driver scheduling">;
 

lib/AST/FineGrainedDependencyFormat.cpp

@@ -44,7 +44,7 @@ class Deserializer {
 
 public:
   Deserializer(llvm::MemoryBufferRef Data) : Cursor(Data) {}
-  bool readFineGrainedDependencyGraph(SourceFileDepGraph &g);
+  bool readFineGrainedDependencyGraph(SourceFileDepGraph &g, Purpose purpose);
   bool readFineGrainedDependencyGraphFromSwiftModule(SourceFileDepGraph &g);
 };
 
@@ -151,14 +151,24 @@ static llvm::Optional<DeclAspect> getDeclAspect(unsigned declAspect) {
   return None;
 }
 
-bool Deserializer::readFineGrainedDependencyGraph(SourceFileDepGraph &g) {
+bool Deserializer::readFineGrainedDependencyGraph(SourceFileDepGraph &g,
+                                                  Purpose purpose) {
   using namespace record_block;
 
-  if (readSignature())
-    return true;
+  switch (purpose) {
+  case Purpose::ForSwiftDeps:
+    if (readSignature())
+      return true;
 
-  if (enterTopLevelBlock())
-    return true;
+    if (enterTopLevelBlock())
+      return true;
+    LLVM_FALLTHROUGH;
+  case Purpose::ForSwiftModule:
+    // N.B. Incremental metadata embedded in swiftmodule files does not have
+    // a leading signature, and its top-level block has already been
+    // consumed by the time we get here.
+    break;
+  }
 
   if (readMetadata())
     return true;
@@ -260,7 +270,7 @@ bool Deserializer::readFineGrainedDependencyGraph(SourceFileDepGraph &g) {
 bool swift::fine_grained_dependencies::readFineGrainedDependencyGraph(
     llvm::MemoryBuffer &buffer, SourceFileDepGraph &g) {
   Deserializer deserializer(buffer.getMemBufferRef());
-  return deserializer.readFineGrainedDependencyGraph(g);
+  return deserializer.readFineGrainedDependencyGraph(g, Purpose::ForSwiftDeps);
 }
 
 bool swift::fine_grained_dependencies::readFineGrainedDependencyGraph(
@@ -322,7 +332,8 @@ class Serializer {
   Serializer(llvm::BitstreamWriter &ExistingOut) : Out(ExistingOut) {}
 
 public:
-  void writeFineGrainedDependencyGraph(const SourceFileDepGraph &g);
+  void writeFineGrainedDependencyGraph(const SourceFileDepGraph &g,
+                                       Purpose purpose);
 };
 
 } // end namespace
@@ -382,11 +393,21 @@ void Serializer::writeMetadata() {
 }
 
 void
-Serializer::writeFineGrainedDependencyGraph(const SourceFileDepGraph &g) {
-  writeSignature();
-  writeBlockInfoBlock();
+Serializer::writeFineGrainedDependencyGraph(const SourceFileDepGraph &g,
+                                            Purpose purpose) {
+  unsigned blockID = 0;
+  switch (purpose) {
+  case Purpose::ForSwiftDeps:
+    writeSignature();
+    writeBlockInfoBlock();
+    blockID = RECORD_BLOCK_ID;
+    break;
+  case Purpose::ForSwiftModule:
+    blockID = INCREMENTAL_INFORMATION_BLOCK_ID;
+    break;
+  }
 
-  llvm::BCBlockRAII restoreBlock(Out, RECORD_BLOCK_ID, 8);
+  llvm::BCBlockRAII restoreBlock(Out, blockID, 8);
 
   using namespace record_block;
 
@@ -468,9 +489,10 @@ unsigned Serializer::getIdentifier(StringRef str) {
 }
 
 void swift::fine_grained_dependencies::writeFineGrainedDependencyGraph(
-    llvm::BitstreamWriter &Out, const SourceFileDepGraph &g) {
+    llvm::BitstreamWriter &Out, const SourceFileDepGraph &g,
+    Purpose purpose) {
   Serializer serializer{Out};
-  serializer.writeFineGrainedDependencyGraph(g);
+  serializer.writeFineGrainedDependencyGraph(g, purpose);
 }
 
 bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath(
@@ -480,7 +502,7 @@ bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath(
   return withOutputFile(diags, path, [&](llvm::raw_ostream &out) {
     SmallVector<char, 0> Buffer;
     llvm::BitstreamWriter Writer{Buffer};
-    writeFineGrainedDependencyGraph(Writer, g);
+    writeFineGrainedDependencyGraph(Writer, g, Purpose::ForSwiftDeps);
     out.write(Buffer.data(), Buffer.size());
     out.flush();
     return false;
@@ -516,7 +538,7 @@ static bool enterTopLevelModuleBlock(llvm::BitstreamCursor &cursor, unsigned ID,
   if (next.Kind != llvm::BitstreamEntry::SubBlock)
     return false;
 
-  if (next.ID == RECORD_BLOCK_ID) {
+  if (next.ID == llvm::bitc::BLOCKINFO_BLOCK_ID) {
     if (shouldReadBlockInfo) {
       if (!cursor.ReadBlockInfoBlock())
         return false;
@@ -531,7 +553,6 @@ static bool enterTopLevelModuleBlock(llvm::BitstreamCursor &cursor, unsigned ID,
     return false;
 
   if (llvm::Error Err = cursor.EnterSubBlock(ID)) {
-    // FIXME this drops the error on the floor.
     consumeError(std::move(Err));
     return false;
   }
@@ -549,18 +570,18 @@ bool swift::fine_grained_dependencies::
 bool Deserializer::readFineGrainedDependencyGraphFromSwiftModule(
     SourceFileDepGraph &g) {
   if (!checkModuleSignature(Cursor, {0xE2, 0x9C, 0xA8, 0x0E}) ||
-      !enterTopLevelModuleBlock(Cursor, RECORD_BLOCK_ID, false)) {
-    return false;
+      !enterTopLevelModuleBlock(Cursor, llvm::bitc::FIRST_APPLICATION_BLOCKID, false)) {
+    return true;
   }
 
   llvm::BitstreamEntry topLevelEntry;
-
+  bool DidNotReadFineGrainedDependencies = true;
   while (!Cursor.AtEndOfStream()) {
     llvm::Expected<llvm::BitstreamEntry> maybeEntry =
         Cursor.advance(llvm::BitstreamCursor::AF_DontPopBlockAtEnd);
     if (!maybeEntry) {
       consumeError(maybeEntry.takeError());
-      return false;
+      return true;
     }
     topLevelEntry = maybeEntry.get();
     if (topLevelEntry.Kind != llvm::BitstreamEntry::SubBlock)
@@ -571,21 +592,25 @@ bool Deserializer::readFineGrainedDependencyGraphFromSwiftModule(
       if (llvm::Error Err =
               Cursor.EnterSubBlock(INCREMENTAL_INFORMATION_BLOCK_ID)) {
         consumeError(std::move(Err));
-        return false;
+        return true;
+      }
+      if (readFineGrainedDependencyGraph(g, Purpose::ForSwiftModule)) {
+        break;
       }
-      readFineGrainedDependencyGraph(g);
+
+      DidNotReadFineGrainedDependencies = false;
       break;
     }
 
     default:
       // Unknown top-level block, possibly for use by a future version of the
       // module format.
       if (Cursor.SkipBlock()) {
-        return false;
+        return true;
       }
       break;
     }
   }
 
-  return false;
+  return DidNotReadFineGrainedDependencies;
 }

lib/Driver/Compilation.cpp

@@ -1161,10 +1161,6 @@ namespace driver {
           continue;
         }
 
-        // Is this module out of date? If not, just keep searching.
-        if (Comp.getLastBuildTime() >= depStatus.getLastModificationTime())
-          continue;
-
         // Can we run a cross-module incremental build at all? If not, fallback.
         if (!Comp.getEnableCrossModuleIncrementalBuild()) {
           fallbackToExternalBehavior(external);

lib/Driver/FineGrainedDependencyDriverGraph.cpp

@@ -120,7 +120,7 @@ ModuleDepGraph::Changes ModuleDepGraph::loadFromSwiftModuleBuffer(
       SourceFileDepGraph::loadFromSwiftModuleBuffer(buffer);
   if (!sourceFileDepGraph)
     return None;
-  registerJob(Cmd);
+  jobsBySwiftDeps[buffer.getBufferIdentifier().str()] = Cmd;
   auto changes = integrate(*sourceFileDepGraph, buffer.getBufferIdentifier());
   if (verifyFineGrainedDependencyGraphAfterEveryImport)
     verify();
@@ -445,11 +445,11 @@ bool ModuleDepGraph::recordWhatUseDependsUpon(
           StringRef externalSwiftDeps = def->getKey().getName();
           if (def->getKey().getKind() == NodeKind::externalDepend) {
             externalDependencies.insert(externalSwiftDeps.str());
+            useHasNewExternalDependency = true;
           } else if (def->getKey().getKind() ==
                      NodeKind::incrementalExternalDepend) {
             incrementalExternalDependencies.insert(externalSwiftDeps.str());
           }
-          useHasNewExternalDependency = true;
         }
       });
   return useHasNewExternalDependency;

lib/Serialization/CMakeLists.txt

@@ -9,7 +9,6 @@ add_swift_host_library(swiftSerialization STATIC
   SerializedModuleLoader.cpp
   SerializedSILLoader.cpp
   SerializeDoc.cpp
-  SerializeIncremental.cpp
   SerializeSIL.cpp
 
   LLVM_LINK_COMPONENTS

lib/Serialization/Serialization.cpp

@@ -861,6 +861,13 @@ void Serializer::writeBlockInfoBlock() {
   BLOCK_RECORD(sil_index_block, SIL_DIFFERENTIABILITY_WITNESS_NAMES);
   BLOCK_RECORD(sil_index_block, SIL_DIFFERENTIABILITY_WITNESS_OFFSETS);
 
+  BLOCK(INCREMENTAL_INFORMATION_BLOCK);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, METADATA);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, SOURCE_FILE_DEP_GRAPH_NODE);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, FINGERPRINT_NODE);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, DEPENDS_ON_DEFINITION_NODE);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, IDENTIFIER_NODE);
+
 #undef BLOCK
 #undef BLOCK_RECORD
 }
@@ -5231,8 +5238,9 @@ void Serializer::writeToStream(
     S.writeInputBlock(options);
     S.writeSIL(SILMod, options.SerializeAllSIL);
     S.writeAST(DC);
-    if (options.ExperimentalCrossModuleIncrementalInfo) {
-      S.writeIncrementalInfo(DepGraph);
+    if (options.ExperimentalCrossModuleIncrementalInfo && DepGraph) {
+      fine_grained_dependencies::writeFineGrainedDependencyGraph(
+          S.Out, *DepGraph, fine_grained_dependencies::Purpose::ForSwiftModule);
     }
   }
 

lib/Serialization/SerializeIncremental.cpp

@@ -973,13 +973,6 @@ SerializedModuleLoaderBase::loadModule(SourceLoc importLoc,
                   isFramework, isSystemModule)) {
     return nullptr;
   }
-  if (dependencyTracker) {
-    // Don't record cached artifacts as dependencies.
-    StringRef DepPath = moduleInputBuffer->getBufferIdentifier();
-    if (!isCached(DepPath)) {
-      dependencyTracker->addDependency(DepPath, /*isSystem=*/false);
-    }
-  }
 
   assert(moduleInputBuffer);
 
@@ -997,6 +990,18 @@ SerializedModuleLoaderBase::loadModule(SourceLoc importLoc,
   } else {
     M->setFailedToLoad();
   }
+
+  if (dependencyTracker && file) {
+    auto DepPath = file->getFilename();
+    // Don't record cached artifacts as dependencies.
+    if (!isCached(DepPath)) {
+      if (M->hasIncrementalInfo()) {
+        dependencyTracker->addIncrementalDependency(DepPath);
+      } else {
+        dependencyTracker->addDependency(DepPath, /*isSystem=*/false);
+      }
+    }
+  }
   return M;
 }
 

lib/Serialization/SerializedModuleLoader.cpp

@@ -0,0 +1,11 @@
+{
+  "A.swift": {
+    "object": "./A.o",
+    "swift-dependencies": "./A.swiftdeps",
+    "swiftmodule": "./A~partial.swiftmodule",
+    "swiftdoc": "./A.swiftdoc",
+  },
+  "": {
+    "swift-dependencies": "./A~buildrecord.swiftdeps"
+  }
+}

test/Incremental/CrossModule/Inputs/linear/A.json

@@ -0,0 +1 @@
+import B

test/Incremental/CrossModule/Inputs/linear/A.swift

@@ -0,0 +1,11 @@
+{
+  "B.swift": {
+    "object": "./B.o",
+    "swift-dependencies": "./B.swiftdeps",
+    "swiftmodule": "./B~partial.swiftmodule",
+    "swiftdoc": "./B.swiftdoc",
+  },
+  "": {
+    "swift-dependencies": "./B~buildrecord.swiftdeps"
+  }
+}

test/Incremental/CrossModule/Inputs/linear/B.json

@@ -0,0 +1,5 @@
+import C
+
+public func fromB() {
+  return fromC()
+}