[5.9 🍒][Dependency Scanning] Consider optional dependencies of @testable textual dependencies with an adjacent binary module

include/swift/AST/ASTContext.h

@@ -1020,6 +1020,8 @@ class ASTContext final {
       StringRef moduleName,
       ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate,
+      bool optionalDependencyLookup = false,
+      bool isTestableImport = false,
       llvm::Optional<std::pair<std::string, swift::ModuleDependencyKind>> dependencyOf = None);
 
   /// Retrieve the module dependencies for the Clang module with the given name.

include/swift/AST/Decl.h

@@ -1422,6 +1422,10 @@ class ImportDecl final : public Decl,
     return getAttrs().hasAttribute<ExportedAttr>();
   }
 
+  bool isTestable() const {
+    return getAttrs().hasAttribute<TestableAttr>();
+  }
+
   ModuleDecl *getModule() const { return Mod; }
   void setModule(ModuleDecl *M) { Mod = M; }
 

include/swift/AST/ModuleDependencies.h

@@ -109,6 +109,11 @@ class ModuleDependencyInfoStorageBase {
   /// The set of modules on which this module depends.
   std::vector<std::string> moduleImports;
 
+  /// The set of modules which constitute optional module
+  /// dependencies for this module, such as `@_implementationOnly`
+  /// or `internal` imports.
+  std::vector<std::string> optionalModuleImports;
+
   /// The set of modules on which this module depends, resolved
   /// to Module IDs, qualified by module kind: Swift, Clang, etc.
   std::vector<ModuleDependencyID> resolvedModuleDependencies;
@@ -206,10 +211,14 @@ class SwiftSourceModuleDependenciesStorage :
   /// Details common to Swift textual (interface or source) modules
   CommonSwiftTextualModuleDependencyDetails textualModuleDetails;
 
+  /// Collection of module imports that were detected to be `@Testable`
+  llvm::StringSet<> testableImports;
+
   SwiftSourceModuleDependenciesStorage(
     ArrayRef<StringRef> extraPCMArgs
   ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource),
-      textualModuleDetails(extraPCMArgs) {}
+      textualModuleDetails(extraPCMArgs),
+      testableImports(llvm::StringSet<>()) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
     return new SwiftSourceModuleDependenciesStorage(*this);
@@ -218,6 +227,10 @@ class SwiftSourceModuleDependenciesStorage :
   static bool classof(const ModuleDependencyInfoStorageBase *base) {
     return base->dependencyKind == ModuleDependencyKind::SwiftSource;
   }
+
+  void addTestableImport(ImportPath::Module module) {
+    testableImports.insert(module.front().Item.str());
+  }
 };
 
 /// Describes the dependencies of a pre-built Swift module (with no .swiftinterface).
@@ -425,6 +438,11 @@ class ModuleDependencyInfo {
     return storage->moduleImports;
   }
 
+  /// Retrieve the module-level optional imports.
+  ArrayRef<std::string> getOptionalModuleImports() const {
+    return storage->optionalModuleImports;
+  }
+
   /// Retreive the module-level dependencies.
   const ArrayRef<ModuleDependencyID> getModuleDependencies() const {
     assert(storage->resolved);
@@ -450,6 +468,13 @@ class ModuleDependencyInfo {
     storage->resolved = isResolved;
   }
 
+  /// For a Source dependency, register a `Testable` import
+  void addTestableImport(ImportPath::Module module);
+
+  /// Whether or not a queried module name is a `@Testable` import dependency
+  /// of this module. Can only return `true` for Swift source modules.
+  bool isTestableImport(StringRef moduleName) const;
+
   /// Whether the dependencies are for a Swift module: either Textual, Source, Binary, or Placeholder.
   bool isSwiftModule() const;
 
@@ -488,6 +513,11 @@ class ModuleDependencyInfo {
   const SwiftPlaceholderModuleDependencyStorage *
     getAsPlaceholderDependencyModule() const;
 
+  /// Add a dependency on the given module, if it was not already in the set.
+  void addOptionalModuleImport(StringRef module,
+                               llvm::StringSet<> *alreadyAddedModules = nullptr);
+
+
   /// Add a dependency on the given module, if it was not already in the set.
   void addModuleImport(StringRef module,
                        llvm::StringSet<> *alreadyAddedModules = nullptr);

include/swift/AST/ModuleLoader.h

@@ -241,7 +241,8 @@ class ModuleLoader {
   /// If a non-null \p versionInfo is provided, the module version will be
   /// parsed and populated.
   virtual bool canImportModule(ImportPath::Module named,
-                               ModuleVersionInfo *versionInfo) = 0;
+                               ModuleVersionInfo *versionInfo,
+                               bool isTestableImport = false) = 0;
 
   /// Import a module with the given module path.
   ///
@@ -324,7 +325,8 @@ class ModuleLoader {
   virtual Optional<const ModuleDependencyInfo*> getModuleDependencies(
       StringRef moduleName,
       ModuleDependenciesCache &cache,
-      InterfaceSubContextDelegate &delegate) = 0;
+      InterfaceSubContextDelegate &delegate,
+      bool isTestableImport = false) = 0;
 };
 
 } // namespace swift

include/swift/ClangImporter/ClangImporter.h

@@ -212,7 +212,8 @@ class ClangImporter final : public ClangModuleLoader {
   /// If a non-null \p versionInfo is provided, the module version will be
   /// parsed and populated.
   virtual bool canImportModule(ImportPath::Module named,
-                               ModuleVersionInfo *versionInfo) override;
+                               ModuleVersionInfo *versionInfo,
+                               bool isTestableImport = false) override;
 
   /// Import a module with the given module path.
   ///
@@ -427,7 +428,8 @@ class ClangImporter final : public ClangModuleLoader {
 
   Optional<const ModuleDependencyInfo*> getModuleDependencies(
       StringRef moduleName, ModuleDependenciesCache &cache,
-      InterfaceSubContextDelegate &delegate) override;
+      InterfaceSubContextDelegate &delegate,
+      bool isTestableImport = false) override;
 
   /// Add dependency information for the bridging header.
   ///

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -142,8 +142,8 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
                   std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-                  bool skipBuildingInterface, bool &isFramework,
-                  bool &isSystemModule) override;
+                  bool skipBuildingInterface, bool isTestableDependencyLookup,
+                  bool &isFramework, bool &isSystemModule) override;
 
   std::error_code findModuleFilesInDirectory(
       ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
@@ -152,10 +152,12 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) override;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup = false) override;
 
   bool canImportModule(ImportPath::Module named,
-                       ModuleVersionInfo *versionInfo) override;
+                       ModuleVersionInfo *versionInfo,
+                       bool isTestableDependencyLookup = false) override;
 
   bool isCached(StringRef DepPath) override { return false; };
 
@@ -476,7 +478,8 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) override;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup = false) override;
 
   bool isCached(StringRef DepPath) override;
 public:

include/swift/Sema/SourceLoader.h

@@ -60,7 +60,8 @@ class SourceLoader : public ModuleLoader {
   /// If a non-null \p versionInfo is provided, the module version will be
   /// parsed and populated.
   virtual bool canImportModule(ImportPath::Module named,
-                               ModuleVersionInfo *versionInfo) override;
+                               ModuleVersionInfo *versionInfo,
+                               bool isTestableDependencyLookup = false) override;
 
   /// Import a module with the given module path.
   ///
@@ -98,7 +99,8 @@ class SourceLoader : public ModuleLoader {
 
   Optional<const ModuleDependencyInfo*>
   getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
-                        InterfaceSubContextDelegate &delegate) override;
+                        InterfaceSubContextDelegate &delegate,
+                        bool isTestableImport) override;
 };
 }
 

include/swift/Serialization/ModuleDependencyScanner.h

@@ -34,7 +34,8 @@ namespace swift {
 
       /// Scan the given interface file to determine dependencies.
       llvm::ErrorOr<ModuleDependencyInfo> scanInterfaceFile(
-          Twine moduleInterfacePath, bool isFramework);
+          Twine moduleInterfacePath, bool isFramework,
+          bool isTestableImport);
 
       InterfaceSubContextDelegate &astDelegate;
 
@@ -61,7 +62,8 @@ namespace swift {
           std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
           std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
           std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-          bool skipBuildingInterface, bool IsFramework) override;
+          bool SkipBuildingInterface, bool IsFramework,
+          bool IsTestableDependencyLookup) override;
 
       virtual void collectVisibleTopLevelModuleNames(
           SmallVectorImpl<Identifier> &names) const override {
@@ -125,8 +127,8 @@ namespace swift {
                  std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
                  std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
                  std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-                 bool skipBuildingInterface, bool &isFramework,
-                 bool &isSystemModule) override;
+                 bool skipBuildingInterface, bool isTestableDependencyLookup,
+                 bool &isFramework, bool &isSystemModule) override;
 
       static bool classof(const ModuleDependencyScanner *MDS) {
         return MDS->getKind() == MDS_placeholder;

include/swift/Serialization/SerializedModuleLoader.h

@@ -20,6 +20,8 @@
 
 namespace swift {
 class ModuleFile;
+class PathObfuscator;
+enum class ModuleLoadingBehavior;
 namespace file_types {
   enum ID : uint8_t;
 }
@@ -84,7 +86,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
       std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-      bool skipBuildingInterface, bool &isFramework, bool &isSystemModule);
+      bool skipBuildingInterface, bool isTestableDependencyLookup,
+      bool &isFramework, bool &isSystemModule);
 
   /// Attempts to search the provided directory for a loadable serialized
   /// .swiftmodule with the provided `ModuleFilename`. Subclasses must
@@ -105,7 +108,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) = 0;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool isTestableDependencyLookup = false) = 0;
 
   std::error_code
   openModuleFile(
@@ -146,6 +150,16 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// Scan the given serialized module file to determine dependencies.
   llvm::ErrorOr<ModuleDependencyInfo> scanModuleFile(Twine modulePath, bool isFramework);
 
+  static llvm::ErrorOr<llvm::StringSet<>>
+  getModuleImportsOfModule(Twine modulePath,
+                           ModuleLoadingBehavior transitiveBehavior,
+                           bool isFramework,
+                           bool isRequiredOSSAModules,
+                           StringRef SDKName,
+                           StringRef packageName,
+                           llvm::vfs::FileSystem *fileSystem,
+                           PathObfuscator &recoverer);
+
   /// Load the module file into a buffer and also collect its module name.
   static std::unique_ptr<llvm::MemoryBuffer>
   getModuleName(ASTContext &Ctx, StringRef modulePath, std::string &Name);
@@ -179,7 +193,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// If a non-null \p versionInfo is provided, the module version will be
   /// parsed and populated.
   virtual bool canImportModule(ImportPath::Module named,
-                               ModuleVersionInfo *versionInfo) override;
+                               ModuleVersionInfo *versionInfo,
+                               bool isTestableDependencyLookup = false) override;
 
   /// Import a module with the given module path.
   ///
@@ -214,7 +229,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
 
   virtual Optional<const ModuleDependencyInfo*> getModuleDependencies(
       StringRef moduleName, ModuleDependenciesCache &cache,
-      InterfaceSubContextDelegate &delegate) override;
+      InterfaceSubContextDelegate &delegate,
+      bool isTestableImport) override;
 };
 
 /// Imports serialized Swift modules into an ASTContext.
@@ -232,7 +248,8 @@ class ImplicitSerializedModuleLoader : public SerializedModuleLoaderBase {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) override;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool isTestableDependencyLookup = false) override;
 
   bool maybeDiagnoseTargetMismatch(
       SourceLoc sourceLocation,
@@ -286,7 +303,8 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) override;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup = false) override;
 
   bool maybeDiagnoseTargetMismatch(
       SourceLoc sourceLocation,
@@ -298,7 +316,8 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
   virtual ~MemoryBufferSerializedModuleLoader();
 
   bool canImportModule(ImportPath::Module named,
-                       ModuleVersionInfo *versionInfo) override;
+                       ModuleVersionInfo *versionInfo,
+                       bool isTestableDependencyLookup = false) override;
 
   ModuleDecl *
   loadModule(SourceLoc importLoc,

lib/AST/ASTContext.cpp

@@ -2009,6 +2009,8 @@ static void diagnoseScannerFailure(StringRef moduleName,
 Optional<const ModuleDependencyInfo*> ASTContext::getModuleDependencies(
     StringRef moduleName, ModuleDependenciesCache &cache,
     InterfaceSubContextDelegate &delegate,
+    bool optionalDependencyLookup,
+    bool isTestableImport,
     llvm::Optional<ModuleDependencyID> dependencyOf) {
   // Retrieve the dependencies for this module.
   // Check whether we've cached this result.
@@ -2030,12 +2032,14 @@ Optional<const ModuleDependencyInfo*> ASTContext::getModuleDependencies(
 
   for (auto &loader : getImpl().ModuleLoaders) {
     if (auto dependencies =
-        loader->getModuleDependencies(moduleName, cache, delegate))
+        loader->getModuleDependencies(moduleName, cache, delegate,
+                                      isTestableImport))
       return dependencies;
   }
 
-  diagnoseScannerFailure(moduleName, Diags, cache,
-                         dependencyOf);
+  if (!optionalDependencyLookup)
+    diagnoseScannerFailure(moduleName, Diags, cache,
+                           dependencyOf);
   return None;
 }