[Dependency Scanning] Handle binary module dependencies' Swift overlay dependencies

The code, previously, only properly handled such dependencies being a distinct category for Swift source and Swift textual dependency infos. Swift binary module dependencies must handle this similarly and this change adds the missing support for them. Recent refactor of the scanner also means that now Swift binary dependencies with Swift overlay dependencies may crash the scanner, and this change resolves this as well.

Resolves rdar://117088840

Author: artemcm

include/swift-c/DependencyScan/DependencyScan.h

@@ -180,6 +180,10 @@ SWIFTSCAN_PUBLIC swiftscan_string_ref_t
 swiftscan_swift_binary_detail_get_module_source_info_path(
     swiftscan_module_details_t details);
 
+SWIFTSCAN_PUBLIC swiftscan_string_set_t *
+swiftscan_swift_binary_detail_get_swift_overlay_dependencies(
+    swiftscan_module_details_t details);
+
 SWIFTSCAN_PUBLIC swiftscan_string_set_t *
 swiftscan_swift_binary_detail_get_header_dependencies(
     swiftscan_module_details_t details);

include/swift/AST/ModuleDependencies.h

@@ -160,6 +160,10 @@ class ModuleDependencyInfoStorageBase {
   /// to Module IDs, qualified by module kind: Swift, Clang, etc.
   std::vector<ModuleDependencyID> resolvedDirectModuleDependencies;
 
+  /// Dependencies comprised of Swift overlay modules of direct and
+  /// transitive Clang dependencies.
+  std::vector<ModuleDependencyID> swiftOverlayDependencies;
+
   /// The cache key for the produced module.
   std::string moduleCacheKey;
 
@@ -192,10 +196,6 @@ struct CommonSwiftTextualModuleDependencyDetails {
   /// (Clang) modules on which the bridging header depends.
   std::vector<std::string> bridgingModuleDependencies;
 
-  /// Dependencies comprised of Swift overlay modules of direct and
-  /// transitive Clang dependencies.
-  std::vector<ModuleDependencyID> swiftOverlayDependencies;
-
   /// The Swift frontend invocation arguments to build the Swift module from the
   /// interface.
   std::vector<std::string> buildCommandLine;
@@ -581,29 +581,12 @@ class ModuleDependencyInfo {
 
   /// Set this module's set of Swift Overlay dependencies
   void setOverlayDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
-    assert(isSwiftSourceModule() || isSwiftInterfaceModule());
-    CommonSwiftTextualModuleDependencyDetails *textualModuleDetails;
-    if (auto sourceDetailsStorage = dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get())) {
-      textualModuleDetails = &sourceDetailsStorage->textualModuleDetails;
-    } else if (auto interfaceDetailsStorage = dyn_cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())) {
-      textualModuleDetails = &interfaceDetailsStorage->textualModuleDetails;
-    } else {
-      llvm_unreachable("Unknown kind of dependency module info.");
-    }
-    textualModuleDetails->swiftOverlayDependencies.assign(dependencyIDs.begin(), dependencyIDs.end());
+    assert(isSwiftModule());
+    storage->swiftOverlayDependencies.assign(dependencyIDs.begin(), dependencyIDs.end());
   }
 
   const ArrayRef<ModuleDependencyID> getSwiftOverlayDependencies() const {
-    CommonSwiftTextualModuleDependencyDetails *textualModuleDetails = nullptr;
-    if (auto sourceDetailsStorage = dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get()))
-      textualModuleDetails = &sourceDetailsStorage->textualModuleDetails;
-    else if (auto interfaceDetailsStorage = dyn_cast<SwiftInterfaceModuleDependenciesStorage>(storage.get()))
-      textualModuleDetails = &interfaceDetailsStorage->textualModuleDetails;
-
-    if (textualModuleDetails)
-      return textualModuleDetails->swiftOverlayDependencies;
-    else
-      return {};
+    return storage->swiftOverlayDependencies;
   }
 
   std::vector<std::string> getCommandline() const {
@@ -1074,7 +1057,7 @@ class ModuleDependenciesCache {
 
   /// Resolve a dependency module's set of Swift module dependencies
   /// that are Swift overlays of Clang module dependencies.
-  void setSwiftOverlayDependencues(ModuleDependencyID moduleID,
+  void setSwiftOverlayDependencies(ModuleDependencyID moduleID,
                                    const ArrayRef<ModuleDependencyID> dependencyIDs);
 
   StringRef getMainModuleName() const {

include/swift/DependencyScan/DependencyScanImpl.h

@@ -121,6 +121,10 @@ typedef struct {
   /// The path to the .swiftSourceInfo file.
   swiftscan_string_ref_t module_source_info_path;
 
+  /// (Swift) module dependencies by means of being overlays of
+  /// Clang module dependencies
+  swiftscan_string_set_t *swift_overlay_module_dependencies;
+
   /// (Clang) header dependencies of this binary module.
   /// Typically pre-compiled bridging header.
   swiftscan_string_set_t *header_dependencies;

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -166,6 +166,7 @@ using SwiftBinaryModuleDetailsLayout =
                    FileIDField,                      // compiledModulePath
                    FileIDField,                      // moduleDocPath
                    FileIDField,                      // moduleSourceInfoPath
+                   DependencyIDArrayIDField,         // swiftOverlayDependencies
                    ImportArrayIDField,               // headerImports
                    IsFrameworkField,                 // isFramework
                    IdentifierIDField                 // moduleCacheKey

lib/AST/ModuleDependencies.cpp

@@ -734,7 +734,7 @@ void ModuleDependenciesCache::resolveDependencyImports(ModuleDependencyID module
   updateDependency(moduleID, dependencyInfo);
 }
 
-void ModuleDependenciesCache::setSwiftOverlayDependencues(ModuleDependencyID moduleID,
+void ModuleDependenciesCache::setSwiftOverlayDependencies(ModuleDependencyID moduleID,
                                                           const ArrayRef<ModuleDependencyID> dependencyIDs) {
   auto optionalDependencyInfo = findDependency(moduleID);
   assert(optionalDependencyInfo.has_value() && "Resolving unknown dependency");

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -474,11 +474,12 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
             "Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record");
       cache.configureForContextHash(getContextHash());
       unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID,
-               headerImportsArrayID, isFramework, moduleCacheKeyID;
+               overlayDependencyIDArrayID, headerImportsArrayID, isFramework,
+               moduleCacheKeyID;
       SwiftBinaryModuleDetailsLayout::readRecord(
           Scratch, compiledModulePathID, moduleDocPathID,
-          moduleSourceInfoPathID, headerImportsArrayID, isFramework,
-          moduleCacheKeyID);
+          moduleSourceInfoPathID, overlayDependencyIDArrayID,
+          headerImportsArrayID, isFramework, moduleCacheKeyID);
 
       auto compiledModulePath = getIdentifier(compiledModulePathID);
       if (!compiledModulePath)
@@ -503,6 +504,12 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
            *currentModuleImports, *currentOptionalModuleImports,
            *headerImports, isFramework, *moduleCacheKey);
 
+      // Add Swift overlay dependencies
+      auto overlayModuleDependencyIDs = getModuleDependencyIDArray(overlayDependencyIDArrayID);
+      if (!overlayModuleDependencyIDs.has_value())
+        llvm::report_fatal_error("Bad overlay dependencies: no qualified dependencies");
+      moduleDep.setOverlayDependencies(overlayModuleDependencyIDs.value());
+
       cache.recordDependency(currentModuleName, std::move(moduleDep),
                              getContextHash());
       hasCurrentModule = false;
@@ -999,6 +1006,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
         getIdentifier(swiftBinDeps->compiledModulePath),
         getIdentifier(swiftBinDeps->moduleDocPath),
         getIdentifier(swiftBinDeps->sourceInfoPath),
+        getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs),
         getArrayID(moduleID, ModuleIdentifierArrayKind::DependencyHeaders),
         swiftBinDeps->isFramework,
         getIdentifier(swiftBinDeps->moduleCacheKey));
@@ -1165,7 +1173,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
                  swiftTextDeps->textualModuleDetails.bridgingModuleDependencies);
         addDependencyIDArray(
             moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs,
-            swiftTextDeps->textualModuleDetails.swiftOverlayDependencies);
+            swiftTextDeps->swiftOverlayDependencies);
         addIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID);
         addIdentifier(swiftTextDeps->textualModuleDetails
                           .CASBridgingHeaderIncludeTreeRootID);
@@ -1181,6 +1189,9 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addIdentifier(swiftBinDeps->moduleCacheKey);
         addStringArray(moduleID, ModuleIdentifierArrayKind::DependencyHeaders,
                        swiftBinDeps->preCompiledBridgingHeaderPaths);
+        addDependencyIDArray(
+            moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs,
+            swiftBinDeps->swiftOverlayDependencies);
         break;
       }
       case swift::ModuleDependencyKind::SwiftPlaceholder: {
@@ -1210,7 +1221,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
             swiftSourceDeps->textualModuleDetails.bridgingModuleDependencies);
         addDependencyIDArray(
             moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs,
-            swiftSourceDeps->textualModuleDetails.swiftOverlayDependencies);
+            swiftSourceDeps->swiftOverlayDependencies);
         addStringArray(
             moduleID, ModuleIdentifierArrayKind::BuildCommandLine,
             swiftSourceDeps->textualModuleDetails.buildCommandLine);

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -525,7 +525,7 @@ ModuleDependencyScanner::resolveDirectModuleDependencies(
   cache.resolveDependencyImports(moduleID, directDependencies.getArrayRef());
   // Resolve the dependency info with Swift overlay dependency information.
   if (!swiftOverlayDependencies.empty())
-    cache.setSwiftOverlayDependencues(moduleID,
+    cache.setSwiftOverlayDependencies(moduleID,
                                       swiftOverlayDependencies.getArrayRef());
 
   ModuleDependencyIDSetVector result = directDependencies;

lib/DependencyScan/ScanDependencies.cpp

@@ -830,6 +830,10 @@ static void writeJSON(llvm::raw_ostream &out,
                              /*indentLevel=*/5,
                              /*trailingComma=*/false);
     } else if (swiftBinaryDeps) {
+      bool hasOverlayDependencies =
+        swiftBinaryDeps->swift_overlay_module_dependencies &&
+        swiftBinaryDeps->swift_overlay_module_dependencies->count > 0;
+
       out << "\"swiftPrebuiltExternal\": {\n";
       assert(swiftBinaryDeps->compiled_module_path.data &&
              get_C_string(swiftBinaryDeps->compiled_module_path)[0] != '\0' &&
@@ -865,6 +869,12 @@ static void writeJSON(llvm::raw_ostream &out,
                              swiftBinaryDeps->header_dependencies, 5,
                              /*trailingComma=*/true);
 
+      if (hasOverlayDependencies) {
+        writeDependencies(out, swiftBinaryDeps->swift_overlay_module_dependencies,
+                          "swiftOverlayDependencies", 5,
+                          /*trailingComma=*/true);
+      }
+
       writeJSONSingleField(out, "isFramework", swiftBinaryDeps->is_framework,
                            5, /*trailingComma=*/false);
     } else {
@@ -1032,7 +1042,7 @@ generateFullDependencyGraph(const CompilerInstance &instance,
         details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_TEXTUAL;
         // Create an overlay dependencies set according to the output format
         std::vector<std::string> bridgedOverlayDependencyNames;
-        bridgeDependencyIDs(swiftTextualDeps->textualModuleDetails.swiftOverlayDependencies,
+        bridgeDependencyIDs(swiftTextualDeps->swiftOverlayDependencies,
                             bridgedOverlayDependencyNames);
 
         details->swift_textual_details = {
@@ -1064,7 +1074,7 @@ generateFullDependencyGraph(const CompilerInstance &instance,
         details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_TEXTUAL;
         // Create an overlay dependencies set according to the output format
         std::vector<std::string> bridgedOverlayDependencyNames;
-        bridgeDependencyIDs(swiftSourceDeps->textualModuleDetails.swiftOverlayDependencies,
+        bridgeDependencyIDs(swiftSourceDeps->swiftOverlayDependencies,
                             bridgedOverlayDependencyNames);
 
         details->swift_textual_details = {
@@ -1096,10 +1106,15 @@ generateFullDependencyGraph(const CompilerInstance &instance,
             create_clone(swiftPlaceholderDeps->sourceInfoPath.c_str())};
       } else if (swiftBinaryDeps) {
         details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_BINARY;
+        // Create an overlay dependencies set according to the output format
+        std::vector<std::string> bridgedOverlayDependencyNames;
+        bridgeDependencyIDs(swiftBinaryDeps->swiftOverlayDependencies,
+                            bridgedOverlayDependencyNames);
         details->swift_binary_details = {
             create_clone(swiftBinaryDeps->compiledModulePath.c_str()),
             create_clone(swiftBinaryDeps->moduleDocPath.c_str()),
             create_clone(swiftBinaryDeps->sourceInfoPath.c_str()),
+            create_set(bridgedOverlayDependencyNames),
             create_set(swiftBinaryDeps->preCompiledBridgingHeaderPaths),
             swiftBinaryDeps->isFramework,
             create_clone(swiftBinaryDeps->moduleCacheKey.c_str())};

test/ScanDependencies/Inputs/CHeaders/ClangModuleWithOverlayedDep.h

@@ -0,0 +1 @@
+#include "F.h"

test/ScanDependencies/Inputs/CHeaders/module.modulemap

@@ -52,4 +52,8 @@ module Y {
   explicit module Private {
     header "Y_Private.h"
   }
+}
+module ClangModuleWithOverlayedDep {
+  header "ClangModuleWithOverlayedDep.h"
+  export *
 }

test/ScanDependencies/module_deps_binary_dep_swift_overlay.swift

@@ -0,0 +1,52 @@
+// REQUIRES: objc_interop
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/clang-module-cache)
+// RUN: %empty-directory(%t/DependencyModules)
+// RUN: split-file --leading-lines %s %t
+
+// Emit a binary module dependency
+// RUN: %target-swift-frontend -emit-module -emit-module-path %t/DependencyModules/BinaryModuleDep.swiftmodule -module-cache-path %t/clang-module-cache %t/BinaryModuleDepSource.swift -module-name BinaryModuleDep -I %S/Inputs/CHeaders -I %S/Inputs/Swift
+
+// Scan the client and ensure both the Client and BinaryModuleDep modules have a Swift overlay dependency on 'F' as imported by 'ClangModuleWithOverlayedDep'
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -I %t/DependencyModules
+// Check the contents of the JSON output
+// RUN: %validate-json %t/deps.json | %FileCheck %s
+
+//--- BinaryModuleDepSource.swift
+import ClangModuleWithOverlayedDep
+
+//--- Client.swift
+import BinaryModuleDep
+
+// CHECK:         "swift": "deps"
+// CHECK:         "directDependencies": [
+// CHECK-DAG:          "swift": "Swift"
+// CHECK-DAG:          "swift": "SwiftOnoneSupport"
+// CHECK-DAG:          "swift": "_Concurrency"
+// CHECK-DAG:          "clang": "_SwiftConcurrencyShims"
+// CHECK-DAG:          "swift": "_StringProcessing"
+// CHECK-DAG:          "clang": "ClangModuleWithOverlayedDep"
+// CHECK-DAG:          "swiftPrebuiltExternal": "BinaryModuleDep"
+// CHECK-DAG:          "swift": "F"
+// CHECK:          ],
+// CHECK:          "swiftOverlayDependencies": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:         "swift": "F"
+// CHECK-NEXT:       }
+// CHECK-NEXT:     ]
+
+// CHECK:      "swiftPrebuiltExternal": "BinaryModuleDep"
+// CHECK:         "directDependencies": [
+// CHECK-DAG:          "swift": "Swift"
+// CHECK-DAG:          "swift": "SwiftOnoneSupport"
+// CHECK-DAG:          "swift": "_Concurrency"
+// CHECK-DAG:          "clang": "_SwiftConcurrencyShims"
+// CHECK-DAG:          "swift": "_StringProcessing"
+// CHECK-DAG:          "clang": "ClangModuleWithOverlayedDep"
+// CHECK-DAG:          "swift": "F"
+// CHECK:          ],
+// CHECK:          "swiftOverlayDependencies": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:         "swift": "F"
+// CHECK-NEXT:       }
+// CHECK-NEXT:     ]

tools/libSwiftScan/libSwiftScan.cpp

@@ -81,6 +81,8 @@ void swiftscan_dependency_info_details_dispose(
         details_impl->swift_textual_details.bridging_source_files);
     swiftscan_string_set_dispose(
         details_impl->swift_textual_details.bridging_module_dependencies);
+    swiftscan_string_set_dispose(
+        details_impl->swift_textual_details.swift_overlay_module_dependencies);
     swiftscan_string_set_dispose(
         details_impl->swift_textual_details.command_line);
     swiftscan_string_set_dispose(
@@ -100,6 +102,10 @@ void swiftscan_dependency_info_details_dispose(
         details_impl->swift_binary_details.module_doc_path);
     swiftscan_string_dispose(
         details_impl->swift_binary_details.module_source_info_path);
+    swiftscan_string_set_dispose(
+        details_impl->swift_binary_details.swift_overlay_module_dependencies);
+    swiftscan_string_set_dispose(
+        details_impl->swift_binary_details.header_dependencies);
     swiftscan_string_dispose(
         details_impl->swift_binary_details.module_cache_key);
     break;
@@ -378,6 +384,12 @@ swiftscan_swift_binary_detail_get_module_source_info_path(
   return details->swift_binary_details.module_source_info_path;
 }
 
+swiftscan_string_set_t *
+swiftscan_swift_binary_detail_get_swift_overlay_dependencies(
+    swiftscan_module_details_t details) {
+  return details->swift_binary_details.swift_overlay_module_dependencies;
+}
+
 swiftscan_string_set_t *
 swiftscan_swift_binary_detail_get_header_dependencies(
     swiftscan_module_details_t details) {

tools/libSwiftScan/libSwiftScan.exports

@@ -22,6 +22,7 @@ swiftscan_swift_textual_detail_get_module_cache_key
 swiftscan_swift_binary_detail_get_compiled_module_path
 swiftscan_swift_binary_detail_get_module_doc_path
 swiftscan_swift_binary_detail_get_module_source_info_path
+swiftscan_swift_binary_detail_get_swift_overlay_dependencies
 swiftscan_swift_binary_detail_get_header_dependencies
 swiftscan_swift_binary_detail_get_is_framework
 swiftscan_swift_binary_detail_get_module_cache_key