Merge pull request #24503 from jrose-apple/artiodactyla

[Serialization] Reduce file size by splitting dep dirnames from basenames

Author: jrose-apple

include/swift/Serialization/ModuleFormat.h

@@ -52,7 +52,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 489; // backing variables
+const uint16_t SWIFTMODULE_VERSION_MINOR = 490; // dependency directories
 
 using DeclIDField = BCFixed<31>;
 
@@ -644,6 +644,7 @@ namespace input_block {
     MODULE_FLAGS, // [unused]
     SEARCH_PATH,
     FILE_DEPENDENCY,
+    DEPENDENCY_DIRECTORY,
     PARSEABLE_INTERFACE_PATH
   };
 
@@ -689,9 +690,15 @@ namespace input_block {
     FileModTimeOrContentHashField, // mtime or content hash (for validation)
     BCFixed<1>,                    // are we reading mtime (0) or hash (1)?
     BCFixed<1>,                    // SDK-relative?
+    BCVBR<8>,                      // subpath-relative index (0=none)
     BCBlob                         // path
   >;
 
+  using DependencyDirectoryLayout = BCRecordLayout<
+    DEPENDENCY_DIRECTORY,
+    BCBlob
+  >;
+
   using ParseableInterfaceLayout = BCRecordLayout<
     PARSEABLE_INTERFACE_PATH,
     BCBlob // file path

lib/Serialization/ModuleFile.cpp

@@ -239,6 +239,9 @@ validateControlBlock(llvm::BitstreamCursor &cursor,
 static bool validateInputBlock(
     llvm::BitstreamCursor &cursor, SmallVectorImpl<uint64_t> &scratch,
     SmallVectorImpl<SerializationOptions::FileDependency> &dependencies) {
+  SmallVector<StringRef, 4> dependencyDirectories;
+  SmallString<256> dependencyFullPathBuffer;
+
   while (!cursor.AtEndOfStream()) {
     auto entry = cursor.advance();
     if (entry.Kind == llvm::BitstreamEntry::EndBlock)
@@ -255,17 +258,30 @@ static bool validateInputBlock(
       bool isHashBased = scratch[2] != 0;
       bool isSDKRelative = scratch[3] != 0;
 
+      StringRef path = blobData;
+      size_t directoryIndex = scratch[4];
+      if (directoryIndex != 0) {
+        if (directoryIndex > dependencyDirectories.size())
+          return true;
+        dependencyFullPathBuffer = dependencyDirectories[directoryIndex-1];
+        llvm::sys::path::append(dependencyFullPathBuffer, blobData);
+        path = dependencyFullPathBuffer;
+      }
+
       if (isHashBased) {
         dependencies.push_back(
           SerializationOptions::FileDependency::hashBased(
-            blobData, isSDKRelative, scratch[0], scratch[1]));
+            path, isSDKRelative, scratch[0], scratch[1]));
       } else {
         dependencies.push_back(
           SerializationOptions::FileDependency::modTimeBased(
-            blobData, isSDKRelative, scratch[0], scratch[1]));
+            path, isSDKRelative, scratch[0], scratch[1]));
       }
       break;
     }
+    case input_block::DEPENDENCY_DIRECTORY:
+      dependencyDirectories.push_back(blobData);
+      break;
     default:
       // Unknown metadata record, possibly for use by a future version of the
       // module format.

lib/Serialization/Serialization.cpp

@@ -835,6 +835,7 @@ void Serializer::writeBlockInfoBlock() {
   BLOCK_RECORD(input_block, MODULE_FLAGS);
   BLOCK_RECORD(input_block, SEARCH_PATH);
   BLOCK_RECORD(input_block, FILE_DEPENDENCY);
+  BLOCK_RECORD(input_block, DEPENDENCY_DIRECTORY);
   BLOCK_RECORD(input_block, PARSEABLE_INTERFACE_PATH);
 
   BLOCK(DECLS_AND_TYPES_BLOCK);
@@ -1062,6 +1063,7 @@ void Serializer::writeInputBlock(const SerializationOptions &options) {
   input_block::ImportedHeaderContentsLayout ImportedHeaderContents(Out);
   input_block::SearchPathLayout SearchPath(Out);
   input_block::FileDependencyLayout FileDependency(Out);
+  input_block::DependencyDirectoryLayout DependencyDirectory(Out);
   input_block::ParseableInterfaceLayout ParseableInterface(Out);
 
   if (options.SerializeOptionsForDebugging) {
@@ -1075,13 +1077,24 @@ void Serializer::writeInputBlock(const SerializationOptions &options) {
       SearchPath.emit(ScratchRecord, /*framework=*/false, /*system=*/false, path);
   }
 
+  // Note: We're not using StringMap here because we don't need to own the
+  // strings.
+  llvm::DenseMap<StringRef, unsigned> dependencyDirectories;
   for (auto const &dep : options.Dependencies) {
+    StringRef directoryName = llvm::sys::path::parent_path(dep.getPath());
+    unsigned &dependencyDirectoryIndex = dependencyDirectories[directoryName];
+    if (!dependencyDirectoryIndex) {
+      // This name must be newly-added. Give it a new ID (and skip 0).
+      dependencyDirectoryIndex = dependencyDirectories.size();
+      DependencyDirectory.emit(ScratchRecord, directoryName);
+    }
     FileDependency.emit(ScratchRecord,
                         dep.getSize(),
                         getRawModTimeOrHash(dep),
                         dep.isHashBased(),
                         dep.isSDKRelative(),
-                        dep.getPath());
+                        dependencyDirectoryIndex,
+                        llvm::sys::path::filename(dep.getPath()));
   }
 
   if (!options.ParseableInterface.empty())

test/ParseableInterface/ModuleCache/SystemDependencies.swiftinterface

@@ -22,6 +22,7 @@
 // RUN: echo 'import SystemDependencies' | %target-swift-frontend -typecheck - -I %S -sdk %t/mock-sdk -module-cache-path %t/MCP-system -emit-dependencies-path %t/dummy.d -track-system-dependencies
 // RUN: test -f %t/MCP-system/SystemDependencies*.swiftmodule
 // RUN: %FileCheck -check-prefix CHECK %s < %t/dummy.d
+// RUN: llvm-bcanalyzer -dump %t/MCP-system/SystemDependencies*.swiftmodule | %FileCheck -check-prefix CHECK-DUMP %s
 // RUN: %{python} %S/Inputs/make-old.py %t/MCP-system/SystemDependencies*.swiftmodule
 
 // Baseline: running the same command again doesn't rebuild the cached module.
@@ -41,6 +42,13 @@
 
 // NEGATIVE-NOT: SomeCModule.h
 // CHECK: SomeCModule.h
+
+// CHECK-DUMP-NOT: usr/include
+// CHECK-DUMP: DEPENDENCY_DIRECTORY{{.+}}'usr/include'
+// CHECK-DUMP-NEXT: FILE_DEPENDENCY{{.+}}'module.modulemap'
+// CHECK-DUMP-NEXT: FILE_DEPENDENCY{{.+}}'SomeCModule.h'
+// CHECK-DUMP-NOT: usr/include
+
 // MODULECACHE-COUNT-2: SystemDependencies-{{[^ ]+}}.swiftmodule
 
 import SomeCModule