Merge pull request #24526 from jrose-apple/5.1-artiodactyla

jrose-apple · web-flow · commit da5fa520834b · 2019-05-06T15:47:34.000-07:00
[5.1] [Serialization] Reduce file size by splitting dep dirnames from basenames
diff --git a/include/swift/Serialization/ModuleFormat.h b/include/swift/Serialization/ModuleFormat.h
@@ -52,7 +52,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 487; // Last change: Opaque result types generic params
+const uint16_t SWIFTMODULE_VERSION_MINOR = 488; // Last change: dependency directories
 
 using DeclIDField = BCFixed<31>;
 
@@ -644,6 +644,7 @@ namespace input_block {
     MODULE_FLAGS, // [unused]
     SEARCH_PATH,
     FILE_DEPENDENCY,
+    DEPENDENCY_DIRECTORY,
     PARSEABLE_INTERFACE_PATH
   };
 
@@ -689,9 +690,15 @@ namespace input_block {
     FileModTimeOrContentHashField, // mtime or content hash (for validation)
     BCFixed<1>,                    // are we reading mtime (0) or hash (1)?
     BCFixed<1>,                    // SDK-relative?
+    BCVBR<8>,                      // subpath-relative index (0=none)
     BCBlob                         // path
   >;
 
+  using DependencyDirectoryLayout = BCRecordLayout<
+    DEPENDENCY_DIRECTORY,
+    BCBlob
+  >;
+
   using ParseableInterfaceLayout = BCRecordLayout<
     PARSEABLE_INTERFACE_PATH,
     BCBlob // file path
diff --git a/lib/Serialization/ModuleFile.cpp b/lib/Serialization/ModuleFile.cpp
@@ -239,6 +239,9 @@ validateControlBlock(llvm::BitstreamCursor &cursor,
 static bool validateInputBlock(
     llvm::BitstreamCursor &cursor, SmallVectorImpl<uint64_t> &scratch,
     SmallVectorImpl<SerializationOptions::FileDependency> &dependencies) {
+  SmallVector<StringRef, 4> dependencyDirectories;
+  SmallString<256> dependencyFullPathBuffer;
+
   while (!cursor.AtEndOfStream()) {
     auto entry = cursor.advance();
     if (entry.Kind == llvm::BitstreamEntry::EndBlock)
@@ -255,17 +258,30 @@ static bool validateInputBlock(
       bool isHashBased = scratch[2] != 0;
       bool isSDKRelative = scratch[3] != 0;
 
+      StringRef path = blobData;
+      size_t directoryIndex = scratch[4];
+      if (directoryIndex != 0) {
+        if (directoryIndex > dependencyDirectories.size())
+          return true;
+        dependencyFullPathBuffer = dependencyDirectories[directoryIndex-1];
+        llvm::sys::path::append(dependencyFullPathBuffer, blobData);
+        path = dependencyFullPathBuffer;
+      }
+
       if (isHashBased) {
         dependencies.push_back(
           SerializationOptions::FileDependency::hashBased(
-            blobData, isSDKRelative, scratch[0], scratch[1]));
+            path, isSDKRelative, scratch[0], scratch[1]));
       } else {
         dependencies.push_back(
           SerializationOptions::FileDependency::modTimeBased(
-            blobData, isSDKRelative, scratch[0], scratch[1]));
+            path, isSDKRelative, scratch[0], scratch[1]));
       }
       break;
     }
+    case input_block::DEPENDENCY_DIRECTORY:
+      dependencyDirectories.push_back(blobData);
+      break;
     default:
       // Unknown metadata record, possibly for use by a future version of the
       // module format.
diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp
@@ -834,6 +834,7 @@ void Serializer::writeBlockInfoBlock() {
   BLOCK_RECORD(input_block, MODULE_FLAGS);
   BLOCK_RECORD(input_block, SEARCH_PATH);
   BLOCK_RECORD(input_block, FILE_DEPENDENCY);
+  BLOCK_RECORD(input_block, DEPENDENCY_DIRECTORY);
   BLOCK_RECORD(input_block, PARSEABLE_INTERFACE_PATH);
 
   BLOCK(DECLS_AND_TYPES_BLOCK);
@@ -1061,6 +1062,7 @@ void Serializer::writeInputBlock(const SerializationOptions &options) {
   input_block::ImportedHeaderContentsLayout ImportedHeaderContents(Out);
   input_block::SearchPathLayout SearchPath(Out);
   input_block::FileDependencyLayout FileDependency(Out);
+  input_block::DependencyDirectoryLayout DependencyDirectory(Out);
   input_block::ParseableInterfaceLayout ParseableInterface(Out);
 
   if (options.SerializeOptionsForDebugging) {
@@ -1074,13 +1076,24 @@ void Serializer::writeInputBlock(const SerializationOptions &options) {
       SearchPath.emit(ScratchRecord, /*framework=*/false, /*system=*/false, path);
   }
 
+  // Note: We're not using StringMap here because we don't need to own the
+  // strings.
+  llvm::DenseMap<StringRef, unsigned> dependencyDirectories;
   for (auto const &dep : options.Dependencies) {
+    StringRef directoryName = llvm::sys::path::parent_path(dep.getPath());
+    unsigned &dependencyDirectoryIndex = dependencyDirectories[directoryName];
+    if (!dependencyDirectoryIndex) {
+      // This name must be newly-added. Give it a new ID (and skip 0).
+      dependencyDirectoryIndex = dependencyDirectories.size();
+      DependencyDirectory.emit(ScratchRecord, directoryName);
+    }
     FileDependency.emit(ScratchRecord,
                         dep.getSize(),
                         getRawModTimeOrHash(dep),
                         dep.isHashBased(),
                         dep.isSDKRelative(),
-                        dep.getPath());
+                        dependencyDirectoryIndex,
+                        llvm::sys::path::filename(dep.getPath()));
   }
 
   if (!options.ParseableInterface.empty())
diff --git a/test/ParseableInterface/ModuleCache/SDKDependencies.swift b/test/ParseableInterface/ModuleCache/SDKDependencies.swift
@@ -12,9 +12,9 @@
 // RUN: llvm-bcanalyzer -dump %t/prebuilt-cache/SdkLib.swiftmodule | %FileCheck %s -check-prefix=PREBUILT
 //
 // PREBUILT: MODULE_BLOCK
-// PREBUILT-NOT: FILE_DEPENDENCY {{.*}}/MCP/{{.*}}
-// PREBUILT-NOT: FILE_DEPENDENCY {{.*}}/prebuilt-cache/{{.*}}
-// PREBUILD-NOT: FILE_DEPENDENCY {{.*}}/lib/swift/{{.*}}
+// PREBUILT-NOT: FILE_DEPENDENCY {{.*[/\\]MCP[/\\]}}
+// PREBUILT-NOT: FILE_DEPENDENCY {{.*[/\\]prebuilt-cache[/\\]}}
+// PREBUILD-NOT: FILE_DEPENDENCY {{.*[/\\]lib[/\\]swift[/\\]}}
 //
 // Re-build them in the opposite order
 // RUN: %empty-directory(%t/prebuilt-cache)
@@ -50,8 +50,8 @@
 // RUN: cat %t/dummy.d | %FileCheck %s -check-prefix=DEPFILE-NEGATIVE
 // RUN: cat %t/dummy.d | %FileCheck %s -check-prefix=DEPFILE
 //
-// DEPFILE-NEGATIVE-NOT: /MCP/
-// DEPFILE-NEGATIVE-NOT: /prebuilt-cache/
+// DEPFILE-NEGATIVE-NOT: {{[/\\]MCP[/\\]}}
+// DEPFILE-NEGATIVE-NOT: {{[/\\]prebuilt-cache[/\\]}}
 //
 // DEPFILE-DAG: SomeCModule.h
 // DEPFILE-DAG: SdkLib.swiftinterface
@@ -79,21 +79,21 @@
 // RUN: cat %t/MCP/SdkLib-*.swiftmodule | %FileCheck %s -check-prefixes=EXLIB,SDKLIB
 //
 // EXLIB: dependencies:
-// EXLIB-DAG: /my-sdk/usr/include/module.modulemap
-// EXLIB-DAG: /my-sdk/usr/include/SomeCModule.h
-// EXLIB-DAG: /my-sdk/ExportedLib.swiftinterface
-// SDKLIB-DAG: /my-sdk/SdkLib.swiftinterface
+// EXLIB-DAG: {{[/\\]my-sdk[/\\]usr[/\\]include[/\\]}}module.modulemap
+// EXLIB-DAG: {{[/\\]my-sdk[/\\]usr[/\\]include[/\\]}}SomeCModule.h
+// EXLIB-DAG: {{[/\\]my-sdk[/\\]}}ExportedLib.swiftinterface
+// SDKLIB-DAG: {{[/\\]my-sdk[/\\]}}SdkLib.swiftinterface
 //
 // Check they don't contain any dependencies from either cache other than themselves
 // RUN: cat %t/MCP/ExportedLib-*.swiftmodule | %FileCheck %s -check-prefix=NOCACHE -DLIB_NAME=ExportedLib
 // RUN: cat %t/MCP/SdkLib-*.swiftmodule | %FileCheck %s -check-prefix=NOCACHE -DLIB_NAME=SdkLib
 //
 // NOCACHE: dependencies:
-// NOCACHE-NOT: /prebuilt-cache/
-// NOCACHE-NOT: /MCP/
-// NOCACHE: /prebuilt-cache/[[LIB_NAME]].swiftmodule
-// NOCACHE-NOT: /prebuilt-cache/
-// NOCACHE-NOT: /MCP/
+// NOCACHE-NOT: {{[/\\]prebuilt-cache[/\\]}}
+// NOCACHE-NOT: {{[/\\]MCP[/\\]}}
+// NOCACHE: {{[/\\]prebuilt-cache[/\\]}}[[LIB_NAME]].swiftmodule
+// NOCACHE-NOT: {{[/\\]prebuilt-cache[/\\]}}
+// NOCACHE-NOT: {{[/\\]MCP[/\\]}}
 //
 // Check we didn't emit anything from the cache in the .d file either
 // RUN: cat %t/dummy.d | %FileCheck %s -check-prefix=DEPFILE-NEGATIVE
@@ -122,10 +122,10 @@
 // RUN: cat %t/MCP/ExportedLib-*.swiftmodule | %FileCheck %s -check-prefix=NEW-EXLIB
 // RUN: cat %t/MCP/SdkLib-*.swiftmodule | %FileCheck %s -check-prefixes=NEW-EXLIB,NEW-SDKLIB
 //
-// NEW-EXLIB-DAG: /my-new-sdk/usr/include/module.modulemap
-// NEW-EXLIB-DAG: /my-new-sdk/usr/include/SomeCModule.h
-// NEW-EXLIB-DAG: /my-new-sdk/ExportedLib.swiftinterface
-// NEW-SDKLIB-DAG: /my-new-sdk/SdkLib.swiftinterface
+// NEW-EXLIB-DAG: {{[/\\]my-new-sdk[/\\]usr[/\\]include[/\\]}}module.modulemap
+// NEW-EXLIB-DAG: {{[/\\]my-new-sdk[/\\]usr[/\\]include[/\\]}}SomeCModule.h
+// NEW-EXLIB-DAG: {{[/\\]my-new-sdk[/\\]}}ExportedLib.swiftinterface
+// NEW-SDKLIB-DAG: {{[/\\]my-new-sdk[/\\]}}SdkLib.swiftinterface
 //
 // Check they don't contain dependencies from the module cache, old prebuilt
 // cache, or new prebuilt cache
diff --git a/test/ParseableInterface/ModuleCache/SystemDependencies.swiftinterface b/test/ParseableInterface/ModuleCache/SystemDependencies.swiftinterface
@@ -22,6 +22,7 @@
 // RUN: echo 'import SystemDependencies' | %target-swift-frontend -typecheck - -I %S -sdk %t/mock-sdk -module-cache-path %t/MCP-system -emit-dependencies-path %t/dummy.d -track-system-dependencies
 // RUN: test -f %t/MCP-system/SystemDependencies*.swiftmodule
 // RUN: %FileCheck -check-prefix CHECK %s < %t/dummy.d
+// RUN: llvm-bcanalyzer -dump %t/MCP-system/SystemDependencies*.swiftmodule | %FileCheck -check-prefix CHECK-DUMP %s
 // RUN: %{python} %S/Inputs/make-old.py %t/MCP-system/SystemDependencies*.swiftmodule
 
 // Baseline: running the same command again doesn't rebuild the cached module.
@@ -35,10 +36,19 @@
 
 // Check that it picked a different cache key.
 // RUN: %empty-directory(%t/MCP-combined)
-// RUN: cp -n %t/MCP/SystemDependencies*.swiftmodule %t/MCP-combined
-// RUN: cp -n %t/MCP-system/SystemDependencies*.swiftmodule %t/MCP-combined
+// RUN: cp %t/MCP/SystemDependencies*.swiftmodule %t/MCP-combined
+// RUN: cp %t/MCP-system/SystemDependencies*.swiftmodule %t/MCP-combined
+// RUN: ls -1 %t/MCP-combined | %FileCheck %s -check-prefix=MODULECACHE
 
 // NEGATIVE-NOT: SomeCModule.h
 // CHECK: SomeCModule.h
 
+// CHECK-DUMP-NOT: usr/include
+// CHECK-DUMP: DEPENDENCY_DIRECTORY{{.+}}'usr/include'
+// CHECK-DUMP-NEXT: FILE_DEPENDENCY{{.+}}'module.modulemap'
+// CHECK-DUMP-NEXT: FILE_DEPENDENCY{{.+}}'SomeCModule.h'
+// CHECK-DUMP-NOT: usr/include
+
+// MODULECACHE-COUNT-2: SystemDependencies-{{[^ ]+}}.swiftmodule
+
 import SomeCModule
