Merge pull request #37585 from artemcm/SerializeModuleDepCache

[Dependency Scanning] Add a binary serialization format for the Inter-Module Dependencies Cache

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -275,8 +275,8 @@ class ModuleDependencies {
 
   /// Describe the module dependencies for a Swift module that can be
   /// built from a Swift interface file (\c .swiftinterface).
-  static ModuleDependencies forSwiftInterface(
-      const std::string &swiftInterfaceFile,
+  static ModuleDependencies forSwiftTextualModule(
+      const Optional<std::string> &swiftInterfaceFile,
       ArrayRef<std::string> compiledCandidates,
       ArrayRef<StringRef> buildCommands,
       ArrayRef<StringRef> extraPCMArgs,
@@ -386,6 +386,9 @@ class ModuleDependencies {
   /// Add a bridging header to a Swift module's dependencies.
   void addBridgingHeader(StringRef bridgingHeader);
 
+  /// Add source files
+  void addSourceFile(StringRef sourceFile);
+
   /// Add source files that the bridging header depends on.
   void addBridgingSourceFile(StringRef bridgingSourceFile);
 

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -0,0 +1,185 @@
+//=== SerializedModuleDependencyCacheFormat.h - serialized format -*- C++-*-=//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_DEPENDENCY_SERIALIZEDCACHEFORMAT_H
+#define SWIFT_DEPENDENCY_SERIALIZEDCACHEFORMAT_H
+
+#include "llvm/Bitcode/BitcodeConvenience.h"
+#include "llvm/Bitstream/BitCodes.h"
+
+namespace llvm {
+class MemoryBuffer;
+}
+
+namespace swift {
+
+class DiagnosticEngine;
+class ModuleDependenciesCache;
+
+namespace dependencies {
+namespace module_dependency_cache_serialization {
+
+using llvm::BCArray;
+using llvm::BCBlob;
+using llvm::BCFixed;
+using llvm::BCRecordLayout;
+using llvm::BCVBR;
+
+/// Every .moddepcache file begins with these 4 bytes, for easy identification.
+const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D',
+                                                                  'C'};
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 1;
+/// Increment this on every change.
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 0;
+
+/// Various identifiers in this format will rely on having their strings mapped
+/// using this ID.
+using IdentifierIDField = BCVBR<13>;
+using FileIDField = IdentifierIDField;
+using ModuleIDField = IdentifierIDField;
+using CompilerFlagField = IdentifierIDField;
+using ContextHashField = IdentifierIDField;
+
+/// A bit that indicates whether or not a module is a framework
+using IsFrameworkField = BCFixed<1>;
+
+/// Arrays of various identifiers, distinguised for readability
+using IdentifierIDArryField = llvm::BCArray<IdentifierIDField>;
+
+/// Identifiers used to refer to the above arrays
+using FileIDArrayIDField = IdentifierIDField;
+using DependencyIDArrayIDField = IdentifierIDField;
+using FlagIDArrayIDField = IdentifierIDField;
+
+/// The ID of the top-level block containing the dependency graph
+const unsigned GRAPH_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID;
+
+/// The .moddepcache file format consists of a METADATA record, followed by
+/// zero or more IDENTIFIER records that contain various strings seen in the graph
+/// (e.g. file names or compiler flags), followed by zero or more IDENTIFIER_ARRAY records
+/// which are arrays of identifiers seen in the graph (e.g. list of source files or list of compile flags),
+/// followed by zero or more MODULE_NODE, *_DETAILS_NODE pairs of records.
+namespace graph_block {
+enum {
+  METADATA = 1,
+  MODULE_NODE,
+  SWIFT_TEXTUAL_MODULE_DETAILS_NODE,
+  SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE,
+  SWIFT_BINARY_MODULE_DETAILS_NODE,
+  CLANG_MODULE_DETAILS_NODE,
+  IDENTIFIER_NODE,
+  IDENTIFIER_ARRAY_NODE
+};
+
+// Always the first record in the file.
+using MetadataLayout = BCRecordLayout<
+    METADATA,    // ID
+    BCFixed<16>, // Inter-Module Dependency graph format major version
+    BCFixed<16>, // Inter-Module Dependency graph format minor version
+    BCBlob       // Compiler version string
+    >;
+
+// After the metadata record, we have zero or more identifier records,
+// for each unique string that is referenced in the graph.
+//
+// Identifiers are referenced by their sequence number, starting from 1.
+// The identifier value 0 is special; it always represents the empty string.
+// There is no IDENTIFIER_NODE serialized that corresponds to it, instead
+// the first IDENTIFIER_NODE always has a sequence number of 1.
+using IdentifierNodeLayout = BCRecordLayout<IDENTIFIER_NODE, BCBlob>;
+
+// After the identifier records we have zero or more identifier array records.
+//
+// These arrays are also referenced by their sequence number,
+// starting from 1, similar to identifiers above. Value 0 indicates an
+// empty array. This record is used because individiual array fields must
+// appear as the last field of whatever record they belong to, and several of
+// the below record layouts contain multiple arrays.
+using IdentifierArrayLayout =
+    BCRecordLayout<IDENTIFIER_ARRAY_NODE, IdentifierIDArryField>;
+
+// After the array records, we have a sequence of Module info
+// records, each of which is followed by one of:
+// - SwiftTextualModuleDetails
+// - SwiftBinaryModuleDetails
+// - SwiftPlaceholderModuleDetails
+// - ClangModuleDetails
+using ModuleInfoLayout =
+    BCRecordLayout<MODULE_NODE,             // ID
+                   IdentifierIDField,       // module name
+                   DependencyIDArrayIDField // directDependencies
+                   >;
+
+using SwiftTextualModuleDetailsLayout =
+    BCRecordLayout<SWIFT_TEXTUAL_MODULE_DETAILS_NODE, // ID
+                   FileIDField,                       // swiftInterfaceFile
+                   FileIDArrayIDField, // compiledModuleCandidates
+                   FlagIDArrayIDField, // buildCommandLine
+                   FlagIDArrayIDField, // extraPCMArgs
+                   ContextHashField,   // contextHash
+                   IsFrameworkField,   // isFramework
+                   FileIDField,        // bridgingHeaderFile
+                   FileIDArrayIDField, // sourceFiles
+                   FileIDArrayIDField, // bridgingSourceFiles
+                   IdentifierIDField   // bridgingModuleDependencies
+                   >;
+
+using SwiftBinaryModuleDetailsLayout =
+    BCRecordLayout<SWIFT_BINARY_MODULE_DETAILS_NODE, // ID
+                   FileIDField,                      // compiledModulePath
+                   FileIDField,                      // moduleDocPath
+                   FileIDField,                      // moduleSourceInfoPath
+                   IsFrameworkField                  // isFramework
+                   >;
+
+using SwiftPlaceholderModuleDetailsLayout =
+    BCRecordLayout<SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE, // ID
+                   FileIDField,                           // compiledModulePath
+                   FileIDField,                           // moduleDocPath
+                   FileIDField // moduleSourceInfoPath
+                   >;
+
+using ClangModuleDetailsLayout =
+    BCRecordLayout<CLANG_MODULE_DETAILS_NODE, // ID
+                   FileIDField,               // moduleMapPath
+                   ContextHashField,          // contextHash
+                   FlagIDArrayIDField,        // commandLine
+                   FileIDArrayIDField         // fileDependencies
+                   >;
+} // namespace graph_block
+
+/// Tries to read the dependency graph from the given buffer.
+/// Returns \c true if there was an error.
+bool readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer,
+                                      ModuleDependenciesCache &cache);
+
+/// Tries to read the dependency graph from the given path name.
+/// Returns true if there was an error.
+bool readInterModuleDependenciesCache(llvm::StringRef path,
+                                      ModuleDependenciesCache &cache);
+
+/// Tries to write the dependency graph to the given path name.
+/// Returns true if there was an error.
+bool writeInterModuleDependenciesCache(DiagnosticEngine &diags,
+                                       llvm::StringRef path,
+                                       const ModuleDependenciesCache &cache);
+
+/// Tries to write out the given dependency cache with the given
+/// bitstream writer.
+void writeInterModuleDependenciesCache(llvm::BitstreamWriter &Out,
+                                       const ModuleDependenciesCache &cache);
+
+} // end namespace module_dependency_cache_serialization
+} // end namespace dependencies
+} // end namespace swift
+
+#endif

include/swift/Frontend/FrontendOptions.h

@@ -304,6 +304,19 @@ class FrontendOptions {
   /// of the main Swift module's source files.
   bool ImportPrescan = false;
 
+  /// After performing a dependency scanning action, serialize the scanner's internal state.
+  bool SerializeDependencyScannerCache = false;
+
+  /// Load and re-use a prior serialized dependency scanner cache.
+  bool ReuseDependencyScannerCache = false;
+
+  /// The path at which to either serialize or deserialize the dependency scanner cache.
+  std::string SerializedDependencyScannerCachePath;
+
+  /// After performing a dependency scanning action, serialize and deserialize the
+  /// dependency cache before producing the result.
+  bool TestDependencyScannerCacheSerialization = false;
+
   /// When performing an incremental build, ensure that cross-module incremental
   /// build metadata is available in any swift modules emitted by this frontend
   /// job.

include/swift/Option/FrontendOptions.td

@@ -185,6 +185,15 @@ def batch_scan_input_file
 def import_prescan : Flag<["-"], "import-prescan">,
    HelpText<"When performing a dependency scan, only dentify all imports of the main Swift module sources">;
 
+def serialize_dependency_scan_cache : Flag<["-"], "serialize-dependency-scan-cache">,
+   HelpText<"After performing a dependency scan, serialize the scanner's internal state.">;
+
+def reuse_dependency_scan_cache : Flag<["-"], "load-dependency-scan-cache">,
+   HelpText<"After performing a dependency scan, serialize the scanner's internal state.">;
+
+def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">,
+  HelpText<"The path to output the dependency scanner's internal state.">;
+
 def enable_copy_propagation : Flag<["-"], "enable-copy-propagation">,
   HelpText<"Run SIL copy propagation to shorten object lifetime.">;
 def disable_copy_propagation : Flag<["-"], "disable-copy-propagation">,
@@ -289,6 +298,9 @@ def debug_crash_immediately : Flag<["-"], "debug-crash-immediately">,
 def debug_crash_after_parse : Flag<["-"], "debug-crash-after-parse">,
   DebugCrashOpt, HelpText<"Force a crash after parsing">;
 
+def debug_test_dependency_scan_cache_serialization: Flag<["-"], "test-dependency-scan-cache-serialization">,
+   HelpText<"After performing a dependency scan, serialize and then deserialize the scanner's internal state.">;
+
 def debugger_support : Flag<["-"], "debugger-support">,
   HelpText<"Process swift code as if running in the debugger">;
 

lib/AST/ModuleDependencies.cpp

@@ -118,6 +118,11 @@ void ModuleDependencies::addBridgingSourceFile(StringRef bridgingSourceFile) {
   swiftStorage->bridgingSourceFiles.push_back(bridgingSourceFile.str());
 }
 
+void ModuleDependencies::addSourceFile(StringRef sourceFile) {
+  auto swiftStorage = cast<SwiftTextualModuleDependenciesStorage>(storage.get());
+  swiftStorage->sourceFiles.push_back(sourceFile.str());
+}
+
 /// Add (Clang) module on which the bridging header depends.
 void ModuleDependencies::addBridgingModuleDependency(
     StringRef module, llvm::StringSet<> &alreadyAddedModules) {

lib/DependencyScan/CMakeLists.txt

@@ -2,6 +2,7 @@
 
 add_swift_host_library(swiftDependencyScan STATIC
   DependencyScanningTool.cpp
+  ModuleDependencyCacheSerialization.cpp
   ScanDependencies.cpp
   StringUtils.cpp)