[Dependency Scanning] Refine cross-import overlay detection algorithm

include/swift/AST/ModuleDependencies.h

@@ -154,12 +154,14 @@ struct ScannerImportStatementInfo {
     uint32_t columnNumber;
   };
 
-  ScannerImportStatementInfo(std::string importIdentifier)
-      : importLocations(), importIdentifier(importIdentifier) {}
+  ScannerImportStatementInfo(std::string importIdentifier, bool isExported)
+      : importLocations(), importIdentifier(importIdentifier),
+        isExported(isExported) {}
 
-  ScannerImportStatementInfo(std::string importIdentifier,
+  ScannerImportStatementInfo(std::string importIdentifier, bool isExported,
                              ImportDiagnosticLocationInfo location)
-      : importLocations({location}), importIdentifier(importIdentifier) {}
+      : importLocations({location}), importIdentifier(importIdentifier),
+        isExported(isExported) {}
 
   void addImportLocation(ImportDiagnosticLocationInfo location) {
     importLocations.push_back(location);
@@ -169,6 +171,8 @@ struct ScannerImportStatementInfo {
   SmallVector<ImportDiagnosticLocationInfo, 4> importLocations;
   /// Imported module string. e.g. "Foo.Bar" in 'import Foo.Bar'
   std::string importIdentifier;
+  /// Is this an @_exported import
+  bool isExported;
 };
 
 /// Base class for the variant storage of ModuleDependencyInfo.
@@ -927,7 +931,7 @@ class ModuleDependencyInfo {
 
   /// Add a dependency on the given module, if it was not already in the set.
   void
-  addOptionalModuleImport(StringRef module,
+  addOptionalModuleImport(StringRef module, bool isExported,
                           llvm::StringSet<> *alreadyAddedModules = nullptr);
 
   /// Add all of the module imports in the given source
@@ -937,13 +941,13 @@ class ModuleDependencyInfo {
                         const SourceManager *sourceManager);
 
   /// Add a dependency on the given module, if it was not already in the set.
-  void addModuleImport(ImportPath::Module module,
+  void addModuleImport(ImportPath::Module module, bool isExported,
                        llvm::StringSet<> *alreadyAddedModules = nullptr,
                        const SourceManager *sourceManager = nullptr,
                        SourceLoc sourceLocation = SourceLoc());
 
   /// Add a dependency on the given module, if it was not already in the set.
-  void addModuleImport(StringRef module,
+  void addModuleImport(StringRef module, bool isExported,
                        llvm::StringSet<> *alreadyAddedModules = nullptr,
                        const SourceManager *sourceManager = nullptr,
                        SourceLoc sourceLocation = SourceLoc());

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -56,10 +56,12 @@ using IsFrameworkField = BCFixed<1>;
 using IsSystemField = BCFixed<1>;
 /// A bit that indicates whether or not a module is that of a static archive
 using IsStaticField = BCFixed<1>;
-/// A bit taht indicates whether or not a link library is a force-load one
+/// A bit that indicates whether or not a link library is a force-load one
 using IsForceLoadField = BCFixed<1>;
-/// A bit taht indicates whether or not an import statement is optional
+/// A bit that indicates whether or not an import statement is optional
 using IsOptionalImport = BCFixed<1>;
+/// A bit that indicates whether or not an import statement is @_exported
+using IsExportedImport = BCFixed<1>;
 
 /// Source location fields
 using LineNumberField = BCFixed<32>;
@@ -176,7 +178,8 @@ using ImportStatementLayout =
                    IdentifierIDField,            // bufferIdentifier
                    LineNumberField,              // lineNumber
                    ColumnNumberField,            // columnNumber
-                   IsOptionalImport              // isOptional
+                   IsOptionalImport,             // isOptional
+                   IsExportedImport              // isExported
                    >;
 using ImportStatementArrayLayout =
     BCRecordLayout<IMPORT_STATEMENT_ARRAY_NODE, IdentifierIDArryField>;

include/swift/Serialization/SerializedModuleLoader.h

@@ -171,6 +171,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
 
   struct BinaryModuleImports {
     llvm::StringSet<> moduleImports;
+    llvm::StringSet<> exportedModules;
     std::string headerImport;
   };
 
@@ -185,7 +186,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
 
   /// If the module has a package name matching the one
   /// specified, return a set of package-only imports for this module.
-  static llvm::ErrorOr<llvm::StringSet<>>
+  static llvm::ErrorOr<std::vector<ScannerImportStatementInfo>>
   getMatchingPackageOnlyImportsOfModule(Twine modulePath,
                                         bool isFramework,
                                         bool isRequiredOSSAModules,

lib/AST/ModuleDependencies.cpp

@@ -117,13 +117,13 @@ bool ModuleDependencyInfo::isTestableImport(StringRef moduleName) const {
 }
 
 void ModuleDependencyInfo::addOptionalModuleImport(
-    StringRef module, llvm::StringSet<> *alreadyAddedModules) {
+    StringRef module, bool isExported, llvm::StringSet<> *alreadyAddedModules) {
   if (!alreadyAddedModules || alreadyAddedModules->insert(module).second)
-    storage->optionalModuleImports.push_back(module.str());
+    storage->optionalModuleImports.push_back({module.str(), isExported});
 }
 
 void ModuleDependencyInfo::addModuleImport(
-    StringRef module, llvm::StringSet<> *alreadyAddedModules,
+    StringRef module, bool isExported, llvm::StringSet<> *alreadyAddedModules,
     const SourceManager *sourceManager, SourceLoc sourceLocation) {
   auto scannerImportLocToDiagnosticLocInfo =
       [&sourceManager](SourceLoc sourceLocation) {
@@ -137,14 +137,16 @@ void ModuleDependencyInfo::addModuleImport(
                              sourceManager->isOwning(sourceLocation);
 
   if (alreadyAddedModules && alreadyAddedModules->contains(module)) {
-    if (validSourceLocation) {
-      // Find a prior import of this module and add import location
-      for (auto &existingImport : storage->moduleImports) {
-        if (existingImport.importIdentifier == module) {
+    // Find a prior import of this module and add import location
+    // and adjust whether or not this module is ever imported as exported
+    for (auto &existingImport : storage->moduleImports) {
+      if (existingImport.importIdentifier == module) {
+        if (validSourceLocation) {
           existingImport.addImportLocation(
-              scannerImportLocToDiagnosticLocInfo(sourceLocation));
-          break;
+            scannerImportLocToDiagnosticLocInfo(sourceLocation));
         }
+        existingImport.isExported |= isExported;
+        break;
       }
     }
   } else {
@@ -153,15 +155,15 @@ void ModuleDependencyInfo::addModuleImport(
 
     if (validSourceLocation)
       storage->moduleImports.push_back(ScannerImportStatementInfo(
-          module.str(), scannerImportLocToDiagnosticLocInfo(sourceLocation)));
+          module.str(), isExported, scannerImportLocToDiagnosticLocInfo(sourceLocation)));
     else
       storage->moduleImports.push_back(
-          ScannerImportStatementInfo(module.str()));
+         ScannerImportStatementInfo(module.str(), isExported));
   }
 }
 
 void ModuleDependencyInfo::addModuleImport(
-    ImportPath::Module module, llvm::StringSet<> *alreadyAddedModules,
+    ImportPath::Module module, bool isExported, llvm::StringSet<> *alreadyAddedModules,
     const SourceManager *sourceManager, SourceLoc sourceLocation) {
   std::string ImportedModuleName = module.front().Item.str().str();
   auto submodulePath = module.getSubmodulePath();
@@ -174,7 +176,7 @@ void ModuleDependencyInfo::addModuleImport(
                               alreadyAddedModules);
   }
 
-  addModuleImport(ImportedModuleName, alreadyAddedModules,
+  addModuleImport(ImportedModuleName, isExported, alreadyAddedModules,
                   sourceManager, sourceLocation);
 }
 
@@ -203,7 +205,8 @@ void ModuleDependencyInfo::addModuleImports(
               importDecl->isExported()))
         continue;
 
-      addModuleImport(realPath, &alreadyAddedModules, sourceManager,
+      addModuleImport(realPath, importDecl->isExported(),
+                      &alreadyAddedModules, sourceManager,
                       importDecl->getLoc());
 
       // Additionally, keep track of which dependencies of a Source

lib/ClangImporter/ClangModuleDependencyScanner.cpp

@@ -281,7 +281,10 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
 
     std::vector<ModuleDependencyID> directDependencyIDs;
     for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
-      dependencies.addModuleImport(moduleName.ModuleName, &alreadyAddedModules);
+      // FIXME: This assumes, conservatively, that all Clang module imports
+      // are exported. We need to fix this once the clang scanner gains the appropriate
+      // API to query this.
+      dependencies.addModuleImport(moduleName.ModuleName, /* isExported */ true, &alreadyAddedModules);
       // It is safe to assume that all dependencies of a Clang module are Clang modules.
       directDependencyIDs.push_back({moduleName.ModuleName, ModuleDependencyKind::Clang});
     }

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -240,10 +240,12 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
        &macroDependencies](ModuleDependencyInfo &moduleDep) {
         // Add imports of this module
         for (const auto &moduleName : currentModuleImports)
-          moduleDep.addModuleImport(moduleName.importIdentifier);
+          moduleDep.addModuleImport(moduleName.importIdentifier,
+                                    moduleName.isExported);
         // Add optional imports of this module
         for (const auto &moduleName : currentOptionalModuleImports)
-          moduleDep.addOptionalModuleImport(moduleName.importIdentifier);
+          moduleDep.addOptionalModuleImport(moduleName.importIdentifier,
+                                            moduleName.isExported);
 
         // Add qualified dependencies of this module
         moduleDep.setImportedClangDependencies(importedClangDependenciesIDs);
@@ -408,10 +410,10 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
     case IMPORT_STATEMENT_NODE: {
       unsigned importIdentifierID, bufferIdentifierID;
       unsigned lineNumber, columnNumber;
-      bool isOptional;
+      bool isOptional, isExported;
       ImportStatementLayout::readRecord(Scratch, importIdentifierID,
                                         bufferIdentifierID, lineNumber,
-                                        columnNumber, isOptional);
+                                        columnNumber, isOptional, isExported);
       auto importIdentifier = getIdentifier(importIdentifierID);
       if (!importIdentifier)
         llvm::report_fatal_error("Bad import statement info: no import name");
@@ -421,7 +423,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
         llvm::report_fatal_error(
             "Bad import statement info: no buffer identifier");
       ImportStatements.push_back(ScannerImportStatementInfo(
-          importIdentifier.value(),
+          importIdentifier.value(), isExported,
           ScannerImportStatementInfo::ImportDiagnosticLocationInfo(
               bufferIdentifier.value(), lineNumber, columnNumber)));
       break;
@@ -1443,7 +1445,7 @@ unsigned ModuleDependenciesCacheSerializer::writeImportStatementInfos(
           Out, ScratchRecord, AbbrCodes[ImportStatementLayout::Code],
           getIdentifier(importInfo.importIdentifier),
           getIdentifier(importLoc.bufferIdentifier), importLoc.lineNumber,
-          importLoc.columnNumber, isOptional);
+          importLoc.columnNumber, isOptional, importInfo.isExported);
       count++;
     }
   };

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -412,6 +412,7 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) {
     // Add any implicit module names.
     for (const auto &import : importInfo.AdditionalUnloadedImports) {
       mainDependencies.addModuleImport(import.module.getModulePath(),
+                                       import.options.contains(ImportFlags::Exported),
                                        &alreadyAddedModules,
                                        &ScanASTContext.SourceMgr);
     }
@@ -420,6 +421,7 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) {
     for (const auto &import : importInfo.AdditionalImports) {
       mainDependencies.addModuleImport(
           import.module.importedModule->getNameStr(),
+          import.options.contains(ImportFlags::Exported),
           &alreadyAddedModules);
     }
 
@@ -432,6 +434,7 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) {
     // add a dependency with the same name to trigger the search.
     if (importInfo.ShouldImportUnderlyingModule) {
       mainDependencies.addModuleImport(mainModule->getName().str(),
+                                       /* isExported */ true,
                                        &alreadyAddedModules);
     }
 
@@ -441,6 +444,7 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) {
     for (const auto &tbdSymbolModule :
          ScanCompilerInvocation.getTBDGenOptions().embedSymbolsFromModules) {
       mainDependencies.addModuleImport(tbdSymbolModule,
+                                       /* isExported */ false,
                                        &alreadyAddedModules);
     }
   }
@@ -586,18 +590,19 @@ static void discoverCrossImportOverlayFiles(
       mainModuleName.str(), ModuleDependencyKind::SwiftSource});
 
   llvm::StringMap<ModuleDependencyIDSet> perSourceFileDependencies;
-  const ModuleDependencyIDSet directSwiftDepsSet{
+  const ModuleDependencyIDSet mainModuleDirectSwiftDepsSet{
       mainModuleInfo.getImportedSwiftDependencies().begin(),
       mainModuleInfo.getImportedSwiftDependencies().end()};
-  const ModuleDependencyIDSet directClangDepsSet{
+  const ModuleDependencyIDSet mainModuleDirectClangDepsSet{
       mainModuleInfo.getImportedClangDependencies().begin(),
       mainModuleInfo.getImportedClangDependencies().end()};
 
   // A utility to map an import identifier to one of the
   // known resolved module dependencies
   auto getModuleIDForImportIdentifier =
-      [directSwiftDepsSet, directClangDepsSet](
-          const std::string &importIdentifierStr) -> ModuleDependencyID {
+      [](const std::string &importIdentifierStr,
+         const ModuleDependencyIDSet &directSwiftDepsSet,
+         const ModuleDependencyIDSet &directClangDepsSet) -> ModuleDependencyID {
     if (auto textualDepIt = directSwiftDepsSet.find(
             {importIdentifierStr, ModuleDependencyKind::SwiftInterface});
         textualDepIt != directSwiftDepsSet.end())
@@ -617,8 +622,9 @@ static void discoverCrossImportOverlayFiles(
   // Collect the set of directly-imported module dependencies
   // for each source file in the source module under scan.
   for (const auto &import : mainModuleInfo.getModuleImports()) {
-    auto importResolvedModuleID =
-        getModuleIDForImportIdentifier(import.importIdentifier);
+    auto importResolvedModuleID = getModuleIDForImportIdentifier(
+        import.importIdentifier, mainModuleDirectSwiftDepsSet,
+        mainModuleDirectClangDepsSet);
     for (const auto &importLocation : import.importLocations)
       perSourceFileDependencies[importLocation.bufferIdentifier].insert(
           importResolvedModuleID);
@@ -636,19 +642,29 @@ static void discoverCrossImportOverlayFiles(
       auto moduleID = worklist.pop_back_val();
       perSourceFileDependencies[bufferIdentifier].insert(moduleID);
       if (isSwiftDependencyKind(moduleID.Kind)) {
-        for (const auto &directSwiftDepID :
-             cache.getImportedSwiftDependencies(moduleID)) {
-          if (perSourceFileDependencies[bufferIdentifier].count(directSwiftDepID))
-            continue;
-          worklist.push_back(directSwiftDepID);
+        auto moduleInfo = cache.findKnownDependency(moduleID);
+        if (llvm::any_of(moduleInfo.getModuleImports(),
+                         [](const ScannerImportStatementInfo &importInfo) {
+                           return importInfo.isExported;
+                         })) {
+          const ModuleDependencyIDSet directSwiftDepsSet{
+              moduleInfo.getImportedSwiftDependencies().begin(),
+              moduleInfo.getImportedSwiftDependencies().end()};
+          const ModuleDependencyIDSet directClangDepsSet{
+              moduleInfo.getImportedClangDependencies().begin(),
+              moduleInfo.getImportedClangDependencies().end()};
+          for (const auto &import : moduleInfo.getModuleImports()) {
+            if (import.isExported) {
+              auto importResolvedDepID = getModuleIDForImportIdentifier(
+                  import.importIdentifier, directSwiftDepsSet,
+                  directClangDepsSet);
+              if (!perSourceFileDependencies[bufferIdentifier].count(
+                      importResolvedDepID))
+                worklist.push_back(importResolvedDepID);
+            }
+          }
         }
       }
-      for (const auto &directClangDepID :
-           cache.getImportedClangDependencies(moduleID)) {
-        if (perSourceFileDependencies[bufferIdentifier].count(directClangDepID))
-          continue;
-        worklist.push_back(directClangDepID);
-      }
     }
   }
 
@@ -1354,7 +1370,8 @@ void ModuleDependencyScanner::resolveCrossImportOverlayDependencies(
      ModuleDependencyInfo::forSwiftSourceModule();
   std::for_each(newOverlays.begin(), newOverlays.end(),
                 [&](Identifier modName) {
-                  dummyMainDependencies.addModuleImport(modName.str());
+                  dummyMainDependencies.addModuleImport(modName.str(),
+                                                        /* isExported */ false);
                 });
 
   // Record the dummy main module's direct dependencies. The dummy main module

lib/Serialization/ScanningLoaders.cpp

@@ -276,6 +276,7 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath,
         auto &imInfo = mainMod->getImplicitImportInfo();
         for (auto import : imInfo.AdditionalUnloadedImports) {
           Result->addModuleImport(import.module.getModulePath(),
+                                  import.options.contains(ImportFlags::Exported),
                                   &alreadyAddedModules, &Ctx.SourceMgr);
         }
 
@@ -301,8 +302,9 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath,
                return adjacentBinaryModulePackageOnlyImports.getError();
 
              for (const auto &requiredImport : *adjacentBinaryModulePackageOnlyImports)
-               if (!alreadyAddedModules.contains(requiredImport.getKey()))
-                 Result->addModuleImport(requiredImport.getKey(),
+               if (!alreadyAddedModules.contains(requiredImport.importIdentifier))
+                 Result->addModuleImport(requiredImport.importIdentifier,
+                                         requiredImport.isExported,
                                          &alreadyAddedModules);
            }
          }