[Dependency Scanning] Resolve cross-import overlays relative to defining interface for prebuilt binary Swift dependencies

When the dependency scanner picks a pre-built binary module candidate for a given dependency, it needs to be able to attempt to resolve its cross-import overlays relative to the textual interface that the binary module was built from. For example, if a collection of binary modules are located in, and resolved as dependencies from, a pre-built module directory, the scanner must lookup their corresponding cross-import overlays relative to the defining interface as read out from the binary module's MODULE_INTERFACE_PATH. #70817 ensures that binary modules serialize the path to their defining textual interface.

Resolves rdar://130778577

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -397,12 +397,14 @@ class SwiftBinaryModuleDependencyStorage
       ArrayRef<ScannerImportStatementInfo> moduleImports,
       ArrayRef<ScannerImportStatementInfo> optionalModuleImports,
       ArrayRef<LinkLibrary> linkLibraries, StringRef headerImport,
-      bool isFramework, bool isStatic, StringRef moduleCacheKey)
+      StringRef definingModuleInterface, bool isFramework, bool isStatic,
+      StringRef moduleCacheKey)
       : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary,
                                         moduleImports, optionalModuleImports,
                                         linkLibraries, moduleCacheKey),
         compiledModulePath(compiledModulePath), moduleDocPath(moduleDocPath),
         sourceInfoPath(sourceInfoPath), headerImport(headerImport),
+        definingModuleInterfacePath(definingModuleInterface),
         isFramework(isFramework), isStatic(isStatic) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
@@ -421,6 +423,10 @@ class SwiftBinaryModuleDependencyStorage
   /// The path of the .h dependency of this module.
   const std::string headerImport;
 
+  /// The path of the defining .swiftinterface that this
+  /// binary .swiftmodule was built from, if one exists.
+  const std::string definingModuleInterfacePath;
+
   /// Source files on which the header inputs depend.
   std::vector<std::string> headerSourceFiles;
 
@@ -433,6 +439,15 @@ class SwiftBinaryModuleDependencyStorage
   /// A flag that indicates this dependency is associated with a static archive
   const bool isStatic;
 
+  /// Return the path to the defining .swiftinterface of this module
+  /// of one was determined. Otherwise, return the .swiftmodule path
+  /// itself.
+  std::string getDefiningModulePath() const {
+    if (definingModuleInterfacePath.empty())
+      return compiledModulePath;
+    return definingModuleInterfacePath;
+  }
+
   static bool classof(const ModuleDependencyInfoStorageBase *base) {
     return base->dependencyKind == ModuleDependencyKind::SwiftBinary;
   }
@@ -588,12 +603,13 @@ class ModuleDependencyInfo {
       ArrayRef<ScannerImportStatementInfo> moduleImports,
       ArrayRef<ScannerImportStatementInfo> optionalModuleImports,
       ArrayRef<LinkLibrary> linkLibraries, StringRef headerImport,
-      bool isFramework, bool isStatic, StringRef moduleCacheKey) {
+      StringRef definingModuleInterface, bool isFramework,
+      bool isStatic, StringRef moduleCacheKey) {
     return ModuleDependencyInfo(
         std::make_unique<SwiftBinaryModuleDependencyStorage>(
             compiledModulePath, moduleDocPath, sourceInfoPath, moduleImports,
-            optionalModuleImports, linkLibraries, headerImport, isFramework,
-            isStatic, moduleCacheKey));
+            optionalModuleImports, linkLibraries, headerImport,
+            definingModuleInterface,isFramework, isStatic, moduleCacheKey));
   }
 
   /// Describe the main Swift module.

lib/AST/ModuleLoader.cpp

@@ -223,7 +223,7 @@ ModuleDependencyInfo::collectCrossImportOverlayNames(
     }
     case swift::ModuleDependencyKind::SwiftBinary: {
       auto *swiftBinaryDep = getAsSwiftBinaryModule();
-      modulePath = swiftBinaryDep->compiledModulePath;
+      modulePath = swiftBinaryDep->getDefiningModulePath();
       assert(modulePath.has_value());
       StringRef parentDir = llvm::sys::path::parent_path(*modulePath);
       if (llvm::sys::path::extension(parentDir) == ".swiftmodule") {

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -514,12 +514,12 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       if (!headerImport)
         llvm::report_fatal_error("Bad binary direct dependencies: no header import");
 
-      // TODO: LinkLibraries
+      // TODO: LinkLibraries, DefiningModulePath
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forSwiftBinaryModule(
            *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
            currentModuleImports, currentOptionalModuleImports, {},
-           *headerImport, isFramework, isStatic, *moduleCacheKey);
+           *headerImport, "", isFramework, isStatic, *moduleCacheKey);
 
       auto headerModuleDependencies = getStringArray(headerModuleDependenciesArrayID);
       if (!headerModuleDependencies)

lib/Serialization/ModuleFile.cpp

@@ -92,19 +92,6 @@ static bool isTargetTooNew(const llvm::Triple &moduleTarget,
   return ctxTarget.isOSVersionLT(moduleTarget);
 }
 
-std::string ModuleFile::resolveModuleDefiningFilename(const ASTContext &ctx) {
-  if (!Core->ModuleInterfacePath.empty()) {
-    std::string interfacePath = Core->ModuleInterfacePath.str();
-    if (llvm::sys::path::is_relative(interfacePath)) {
-      SmallString<128> absoluteInterfacePath(ctx.SearchPathOpts.getSDKPath());
-      llvm::sys::path::append(absoluteInterfacePath, interfacePath);
-      return absoluteInterfacePath.str().str();
-    } else
-      return interfacePath;
-  } else
-    return getModuleLoadedFilename().str();
-}
-
 namespace swift {
 namespace serialization {
 bool areCompatible(const llvm::Triple &moduleTarget,
@@ -272,7 +259,8 @@ Status ModuleFile::associateWithFileContext(FileUnit *file, SourceLoc diagLoc,
 
   ASTContext &ctx = getContext();
   // Resolve potentially-SDK-relative module-defining .swiftinterface path
-  ResolvedModuleDefiningFilename = resolveModuleDefiningFilename(ctx);
+  ResolvedModuleDefiningFilename =
+       Core->resolveModuleDefiningFilePath(ctx.SearchPathOpts.getSDKPath());
 
   llvm::Triple moduleTarget(llvm::Triple::normalize(Core->TargetTriple));
   if (!areCompatible(moduleTarget, ctx.LangOpts.Target)) {

lib/Serialization/ModuleFile.h

@@ -285,10 +285,6 @@ class ModuleFile
   ArrayRef<ProtocolConformanceID>
   claimLazyConformanceLoaderToken(uint64_t token);
 
-  /// If the module-defining `.swiftinterface` file is an SDK-relative path,
-  /// resolve it to be absolute to the context's SDK.
-  std::string resolveModuleDefiningFilename(const ASTContext &ctx);
-
   /// Represents an identifier that may or may not have been deserialized yet.
   ///
   /// If \c Ident is empty, the identifier has not been loaded yet.

lib/Serialization/ModuleFileSharedCore.cpp

@@ -1785,6 +1785,19 @@ bool ModuleFileSharedCore::hasSourceInfo() const {
   return !!DeclUSRsTable;
 }
 
+std::string ModuleFileSharedCore::resolveModuleDefiningFilePath(const StringRef SDKPath) const {
+  if (!ModuleInterfacePath.empty()) {
+    std::string interfacePath = ModuleInterfacePath.str();
+    if (llvm::sys::path::is_relative(interfacePath)) {
+      SmallString<128> absoluteInterfacePath(SDKPath);
+      llvm::sys::path::append(absoluteInterfacePath, interfacePath);
+      return absoluteInterfacePath.str().str();
+    } else
+      return interfacePath;
+  } else
+    return ModuleInputBuffer->getBufferIdentifier().str();
+}
+
 ModuleLoadingBehavior
 ModuleFileSharedCore::getTransitiveLoadingBehavior(
                                           const Dependency &dependency,

lib/Serialization/ModuleFileSharedCore.h

@@ -623,6 +623,10 @@ class ModuleFileSharedCore {
     return Bits.IsStaticLibrary;
   }
 
+  /// If the module-defining `.swiftinterface` file is an SDK-relative path,
+  /// resolve it to be absolute to the specified SDK.
+  std::string resolveModuleDefiningFilePath(const StringRef SDKPath) const;
+
   /// Returns \c true if this module file contains a section with incremental
   /// information.
   bool hasIncrementalInfo() const { return HasIncrementalInfo; }

lib/Serialization/SerializedModuleLoader.cpp

@@ -511,11 +511,16 @@ SerializedModuleLoaderBase::scanModuleFile(Twine modulePath, bool isFramework,
                                           LibraryKind::Framework));
   }
 
+  // Attempt to resolve the module's defining .swiftinterface path
+  std::string definingModulePath =
+       loadedModuleFile->resolveModuleDefiningFilePath(Ctx.SearchPathOpts.getSDKPath());
+
   // Map the set of dependencies over to the "module dependencies".
   auto dependencies = ModuleDependencyInfo::forSwiftBinaryModule(
       modulePath.str(), moduleDocPath, sourceInfoPath, moduleImports,
-      optionalModuleImports, linkLibraries, importedHeader, isFramework,
-      loadedModuleFile->isStaticLibrary(), /*module-cache-key*/ "");
+      optionalModuleImports, linkLibraries, importedHeader,
+      definingModulePath, isFramework, loadedModuleFile->isStaticLibrary(),
+      /*module-cache-key*/ "");
 
   return std::move(dependencies);
 }

test/ScanDependencies/module_deps_cross_import_of_binary_module.swift

@@ -0,0 +1,85 @@
+// REQUIRES: objc_interop
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/module-cache)
+// RUN: %empty-directory(%t/inputs)
+// RUN: %empty-directory(%t/binaryInputs)
+// RUN: %empty-directory(%t/separateModules)
+// RUN: %empty-directory(%t/inputs/Foo.swiftcrossimport)
+// RUN: split-file %s %t
+
+// - Fixup the input module file map
+// RUN: sed -e "s|INPUTSDIR|%/t/inputs|g" %t/map.json.template > %t/map.json.template1
+// RUN: sed -e "s|STDLIBMOD|%/stdlib_module|g" %t/map.json.template1 > %t/map.json.template2
+// RUN: sed -e "s|ONONEMOD|%/ononesupport_module|g" %t/map.json.template2 > %t/map.json.template3
+// RUN: sed -e "s|SWIFTLIBDIR|%swift-lib-dir|g" %t/map.json.template3 > %t/map.json
+
+// - Pre-compile explicit module dependency inputs
+// RUN: %target-swift-emit-pcm -module-name SwiftShims %swift-lib-dir/swift/shims/module.modulemap -o %t/inputs/SwiftShims.pcm
+
+// - Pre-compile the Foo module into a separately-stored binary module
+// RUN: %target-swift-frontend -compile-module-from-interface %t/separateModules/Foo.swiftinterface -o %t/binaryInputs/Foo.swiftmodule -module-name Foo -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -disable-implicit-swift-modules -Xcc -fno-implicit-modules -Xcc -fno-implicit-module-maps -explicit-swift-module-map-file %t/map.json
+
+// - Run a dependency scan on test.swift which will pick-up the ready-made binary dependency on Foo.swiftmodule
+//   and use the binary module's serialized originating defining .swiftinterface path to be able to
+//   discover the cross-import overlay _Foo_Bar.
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/module-cache %t/test.swift -o %t/deps.json -I %t/inputs -I %t/binaryInputs -module-name test -enable-cross-import-overlays -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import
+
+// CHECK: "mainModuleName": "test"
+// CHECK:        "swift": "test"
+// CHECK-NEXT: },
+// CHECK-NEXT: {
+// CHECK-NEXT:   "modulePath": "test.swiftmodule"
+// CHECK-NEXT:   "sourceFiles": [
+// CHECK-NEXT:   "{{.*}}{{/|\\}}module_deps_cross_import_of_binary_module.swift.tmp{{/|\\}}test.swift"
+// CHECK-NEXT:   ]
+// CHECK-NEXT:   "directDependencies": [
+// CHECK-DAG:      "swiftPrebuiltExternal": "Swift"
+// CHECK-DAG:      "swiftPrebuiltExternal": "SwiftOnoneSupport"
+// CHECK-DAG:      "swiftPrebuiltExternal": "Foo"
+// CHECK-DAG:      "swift": "Bar"
+// CHECK-DAG:      "swift": "_Foo_Bar"
+
+//--- map.json.template
+[
+  {
+      "moduleName": "Swift",
+      "modulePath": "STDLIBMOD",
+      "isFramework": false
+  },
+  {
+      "moduleName": "SwiftOnoneSupport",
+      "modulePath": "ONONEMOD",
+      "isFramework": false
+  },
+  {
+      "moduleName": "SwiftShims",
+      "isFramework": false,
+      "clangModuleMapPath": "SWIFTLIBDIR/swift/shims/module.modulemap",
+      "clangModulePath": "INPUTSDIR/SwiftShims.pcm"
+}]
+
+//--- test.swift
+import Foo
+import Bar
+
+//--- separateModules/Foo.swiftinterface
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name Foo
+public func foo() {}
+
+//--- inputs/Bar.swiftinterface
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name Bar
+public func bar() {}
+
+//--- inputs/_Foo_Bar.swiftinterface
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name _Foo_Bar
+public func foobar() {}
+
+//--- separateModules/Foo.swiftcrossimport/Bar.swiftoverlay
+%YAML 1.2
+---
+version: 1
+modules:
+  - name: _Foo_Bar