Merge pull request #29582 from brentdax/the-most-ambitious-crossover-event-in-history

Cross-Import Overlays

Author: beccadax

docs/Lexicon.rst

@@ -264,10 +264,32 @@ source code, tests, and commit messages. See also the `LLVM lexicon`_.
     compiler can do something with it.
 
   overlay
-    A library that is imported whenever a C library or framework by the same
-    name is imported. The purpose of an overlay is to augment and extend a
-    library on the system when the library on the system cannot be modified.
-    Apple has a number of overlays for its own SDKs in stdlib/public/SDK/.
+    A wrapper library that is implicitly imported "on top of" another library.
+    It contains an @_exported import of the underlying library, but it augments
+    it with additional APIs which, for one reason or another, are not included
+    in the underlying library directly.
+
+    There are two kinds of overlays:
+
+    A "clang overlay" (the older kind, so it's often just called an "overlay")
+    is a Swift library that adds Swift-specific functionality to a C-family
+    library or framework. Clang overlays are used with system libraries that
+    cannot be modified to add Swift features. A clang overlay has the same
+    module name as the underlying library and can do a few special things that
+    normal modules can't, like adding required initializers to classes. If a
+    module has a clang overlay, the Clang Importer will always load it unless it
+    is actually compiling the overlay itself. Apple has a number of clang
+    overlays for its own SDKs in stdlib/public/Darwin/.
+
+    A "separately-imported overlay" is a separate module with its own name.
+    Unlike a clang overlay, it can be imported in some SourceFiles and not
+    others. When the compiler processes import declarations, it determines which
+    separately-imported overlays need to be imported and then adds them to the
+    list of imports for that file; name lookup also knows to look through the
+    overlay when it looks for declarations in the underlying module.
+    Separately-imported overlays are used to implement the "cross-import
+    overlays" feature, which is used to augment a module with additional
+    functionality when it is imported alongside another module.
 
   PCH
     Precompiled header, a type of file ending in .pch. A precompiled header is

include/swift/AST/Attr.h

@@ -2103,6 +2103,15 @@ class DeclAttributes {
     return nullptr;
   }
 
+  /// Retrieve the first attribute with the given kind.
+  DeclAttribute *getAttribute(DeclAttrKind DK,
+                              bool AllowInvalid = false) {
+    for (auto Attr : *this)
+      if (Attr->getKind() == DK && (Attr->isValid() || AllowInvalid))
+        return Attr;
+    return nullptr;
+  }
+
 private:
   /// Predicate used to filter MatchingAttributeRange.
   template <typename ATTR, bool AllowInvalid> struct ToAttributeKind {

include/swift/AST/DeclContext.h

@@ -578,6 +578,14 @@ class alignas(1 << DeclContextAlignInBits) DeclContext {
                        SmallVectorImpl<ConformanceDiagnostic> *diagnostics
                          = nullptr) const;
 
+  /// Retrieves a list of separately imported overlays which are shadowing
+  /// \p declaring. If any \p overlays are returned, qualified lookups into
+  /// \p declaring should be performed into \p overlays instead; since they
+  /// are overlays, they will re-export \p declaring, but will also augment it
+  /// with additional symbols.
+  void getSeparatelyImportedOverlays(
+      ModuleDecl *declaring, SmallVectorImpl<ModuleDecl *> &overlays) const;
+
   /// Retrieve the syntactic depth of this declaration context, i.e.,
   /// the number of non-module-scoped contexts.
   ///

include/swift/AST/DiagnosticsCommon.def

@@ -165,6 +165,21 @@ WARNING(warn_property_wrapper_module_scope,none,
         "wrapper %0; please qualify the reference with %1",
         (DeclNameRef, Identifier))
 
+//------------------------------------------------------------------------------
+// MARK: Cross-import overlay loading diagnostics
+//------------------------------------------------------------------------------
+ERROR(cannot_load_swiftoverlay_file, none,
+      "cannot load cross-import overlay for %0 and %1: %2 (declared by '%3')",
+      (Identifier, Identifier, StringRef, StringRef))
+ERROR(cannot_list_swiftcrossimport_dir, none,
+      "cannot list cross-import overlays for %0: %1 (declared in '%2')",
+      (Identifier, StringRef, StringRef))
+WARNING(cross_imported_by_both_modules, none,
+        "modules %0 and %1 both declare module %2 as a cross-import overlay, "
+        "which may cause paradoxical behavior when looking up names in them; "
+        "please report this bug to the maintainers of these modules",
+        (Identifier, Identifier, Identifier))
+
 #ifndef DIAG_NO_UNDEF
 # if defined(DIAG)
 #  undef DIAG

include/swift/AST/FileUnit.h

@@ -214,6 +214,13 @@ class FileUnit : public DeclContext {
   virtual void
   collectLinkLibraries(ModuleDecl::LinkLibraryCallback callback) const {}
 
+  /// Returns the path of the file or directory that defines the module
+  /// represented by this \c FileUnit, or empty string if there is none.
+  /// Cross-import overlay specifiers are found relative to this path.
+  virtual StringRef getModuleDefiningPath() const {
+    return "";
+  }
+
   /// True if this file contains the main class for the module.
   bool hasMainClass() const {
     return getMainClass();

include/swift/AST/Module.h

@@ -124,6 +124,8 @@ enum class ResilienceStrategy : unsigned {
   Resilient
 };
 
+class OverlayFile;
+
 /// The minimum unit of compilation.
 ///
 /// A module is made up of several file-units, which are all part of the same
@@ -203,6 +205,9 @@ class ModuleDecl : public DeclContext, public TypeDecl {
 
   SmallVector<FileUnit *, 2> Files;
 
+  llvm::SmallDenseMap<Identifier, SmallVector<OverlayFile *, 1>>
+    declaredCrossImports;
+
   std::unique_ptr<SourceLookupCache> Cache;
   SourceLookupCache &getSourceLookupCache() const;
 
@@ -252,6 +257,22 @@ class ModuleDecl : public DeclContext, public TypeDecl {
   void addFile(FileUnit &newFile);
   void removeFile(FileUnit &existingFile);
 
+  /// Add a file declaring a cross-import overlay.
+  void addCrossImportOverlayFile(StringRef file);
+
+  /// Append to \p overlayNames the names of all modules that this module
+  /// declares should be imported when \p bystanderName is imported.
+  ///
+  /// This operation is asymmetric: you will get different results if you
+  /// reverse the positions of the two modules involved in the cross-import.
+  void findDeclaredCrossImportOverlays(
+      Identifier bystanderName, SmallVectorImpl<Identifier> &overlayNames,
+      SourceLoc diagLoc) const;
+
+  /// Get the list of all modules this module declares a cross-import with.
+  void getDeclaredCrossImportBystanders(
+      SmallVectorImpl<Identifier> &bystanderNames);
+
   /// Convenience accessor for clients that know what kind of file they're
   /// dealing with.
   SourceFile &getMainSourceFile(SourceFileKind expectedKind) const;
@@ -444,7 +465,11 @@ class ModuleDecl : public DeclContext, public TypeDecl {
     /// Include "regular" imports with no special annotation.
     Private = 1 << 1,
     /// Include imports declared with `@_implementationOnly`.
-    ImplementationOnly = 1 << 2
+    ImplementationOnly = 1 << 2,
+    /// Include imports shadowed by a separately-imported overlay (i.e. a
+    /// cross-import overlay). Unshadowed imports are included whether or not
+    /// this flag is specified.
+    ShadowedBySeparateOverlay = 1 << 4
   };
   /// \sa getImportedModules
   using ImportFilter = OptionSet<ImportFilterKind>;

include/swift/AST/ModuleLoader.h

@@ -38,6 +38,7 @@ namespace swift {
 class AbstractFunctionDecl;
 class ClangImporterOptions;
 class ClassDecl;
+class FileUnit;
 class ModuleDecl;
 class NominalTypeDecl;
 class TypeDecl;
@@ -154,6 +155,10 @@ class ModuleLoader {
 
   /// Verify all modules loaded by this loader.
   virtual void verifyAllModules() { }
+
+  /// Discover overlays declared alongside this file and add infomation about
+  /// them to it.
+  void findOverlayFiles(SourceLoc diagLoc, ModuleDecl *module, FileUnit *file);
 };
 
 } // namespace swift

include/swift/AST/SourceFile.h

@@ -54,7 +54,10 @@ class SourceFile final : public FileUnit {
     /// elsewhere.
     ///
     /// Mutually exclusive with Exported.
-    ImplementationOnly = 0x8
+    ImplementationOnly = 0x8,
+
+    /// Used for DenseMap.
+    Reserved = 0x80
   };
 
   /// \see ImportFlags
@@ -69,7 +72,8 @@ class SourceFile final : public FileUnit {
                        StringRef filename = {})
         : module(module), importOptions(options), filename(filename) {
       assert(!(importOptions.contains(ImportFlags::Exported) &&
-               importOptions.contains(ImportFlags::ImplementationOnly)));
+               importOptions.contains(ImportFlags::ImplementationOnly)) ||
+             importOptions.contains(ImportFlags::Reserved));
     }
   };
 
@@ -128,6 +132,17 @@ class SourceFile final : public FileUnit {
   /// The list of top-level declarations in the source file.
   std::vector<Decl *> Decls;
 
+  using SeparatelyImportedOverlayMap =
+    llvm::SmallDenseMap<ModuleDecl *, llvm::SmallPtrSet<ModuleDecl *, 1>>;
+
+  /// Keys are modules which are shadowed by one or more separately-imported
+  /// overlays; values are the list of overlays shadowing them.
+  ///
+  /// This is used by cross-import overlays to make their members appear to
+  /// be part of the underlying module. (ClangImporter overlays use a different
+  /// mechanism which is not SourceFile-dependent.)
+  SeparatelyImportedOverlayMap separatelyImportedOverlays;
+
   friend ASTContext;
   friend Impl;
 
@@ -257,6 +272,31 @@ class SourceFile final : public FileUnit {
 
   bool isImportedImplementationOnly(const ModuleDecl *module) const;
 
+  bool shouldCrossImport() const;
+
+  /// Register a separately-imported overlay as shadowing the module that
+  /// declares it.
+  ///
+  /// \returns true if the overlay was added; false if it already existed.
+  bool addSeparatelyImportedOverlay(ModuleDecl *overlay,
+                                    ModuleDecl *declaring) {
+    return std::get<1>(separatelyImportedOverlays[declaring].insert(overlay));
+  }
+
+  /// Retrieves a list of separately imported overlays which are shadowing
+  /// \p declaring. If any \p overlays are returned, qualified lookups into
+  /// \p declaring should be performed into \p overlays instead; since they
+  /// are overlays, they will re-export \p declaring, but will also augment it
+  /// with additional symbols.
+  void getSeparatelyImportedOverlays(
+      ModuleDecl *declaring, SmallVectorImpl<ModuleDecl *> &overlays) {
+    auto i = separatelyImportedOverlays.find(declaring);
+    if (i == separatelyImportedOverlays.end()) return;
+
+    auto &value = std::get<1>(*i);
+    overlays.append(value.begin(), value.end());
+  }
+
   void cacheVisibleDecls(SmallVectorImpl<ValueDecl *> &&globals) const;
   const SmallVectorImpl<ValueDecl *> &getCachedVisibleDecls() const;
 
@@ -508,4 +548,58 @@ inline void simple_display(llvm::raw_ostream &out, const SourceFile *SF) {
 }
 } // end namespace swift
 
+namespace llvm {
+
+template<>
+struct DenseMapInfo<swift::SourceFile::ImportOptions> {
+  using ImportOptions = swift::SourceFile::ImportOptions;
+
+  using UnsignedDMI = DenseMapInfo<uint8_t>;
+
+  static inline ImportOptions getEmptyKey() {
+    return ImportOptions(UnsignedDMI::getEmptyKey());
+  }
+  static inline ImportOptions getTombstoneKey() {
+    return ImportOptions(UnsignedDMI::getTombstoneKey());
+  }
+  static inline unsigned getHashValue(ImportOptions options) {
+    return UnsignedDMI::getHashValue(options.toRaw());
+  }
+  static bool isEqual(ImportOptions a, ImportOptions b) {
+    return UnsignedDMI::isEqual(a.toRaw(), b.toRaw());
+  }
+};
+
+template<>
+struct DenseMapInfo<swift::SourceFile::ImportedModuleDesc> {
+  using ImportedModuleDesc = swift::SourceFile::ImportedModuleDesc;
+
+  using ImportedModuleDMI = DenseMapInfo<swift::ModuleDecl::ImportedModule>;
+  using ImportOptionsDMI = DenseMapInfo<swift::SourceFile::ImportOptions>;
+  using StringRefDMI = DenseMapInfo<StringRef>;
+
+  static inline ImportedModuleDesc getEmptyKey() {
+    return ImportedModuleDesc(ImportedModuleDMI::getEmptyKey(),
+                              ImportOptionsDMI::getEmptyKey(),
+                              StringRefDMI::getEmptyKey());
+  }
+  static inline ImportedModuleDesc getTombstoneKey() {
+    return ImportedModuleDesc(ImportedModuleDMI::getTombstoneKey(),
+                              ImportOptionsDMI::getTombstoneKey(),
+                              StringRefDMI::getTombstoneKey());
+  }
+  static inline unsigned getHashValue(const ImportedModuleDesc &import) {
+    return combineHashValue(ImportedModuleDMI::getHashValue(import.module),
+           combineHashValue(ImportOptionsDMI::getHashValue(import.importOptions),
+                            StringRefDMI::getHashValue(import.filename)));
+  }
+  static bool isEqual(const ImportedModuleDesc &a,
+                      const ImportedModuleDesc &b) {
+    return ImportedModuleDMI::isEqual(a.module, b.module) &&
+           ImportOptionsDMI::isEqual(a.importOptions, b.importOptions) &&
+           StringRefDMI::isEqual(a.filename, b.filename);
+  }
+};
+}
+
 #endif

include/swift/Basic/FileTypes.def

@@ -73,6 +73,13 @@ TYPE("module-trace",        ModuleTrace,               "trace.json",      "")
 TYPE("index-data",          IndexData,                 "",                "")
 TYPE("opt-record",          OptRecord,                 "opt.yaml",        "")
 
+// Overlay files declare wrapper modules, called "separately-imported overlays",
+// that should be automatically imported when a particular module is imported.
+// Cross-import directories conditionalize overlay files so they only take
+// effect when certain other modules are also loaded.
+TYPE("swiftcrossimport",    SwiftCrossImportDir,       "swiftcrossimport","")
+TYPE("swiftoverlay",        SwiftOverlayFile,          "swiftoverlay",    "")
+
 // Misc types
 TYPE("pcm",                 ClangModuleFile,           "pcm",             "")
 TYPE("pch",                 PCH,                       "pch",             "")

include/swift/Basic/LangOptions.h

@@ -109,6 +109,9 @@ namespace swift {
     /// human-readable string.
     bool EnableConcisePoundFile = false;
 
+    /// Detect and automatically import modules' cross-import overlays.
+    bool EnableCrossImportOverlays = false;
+
     ///
     /// Support for alternate usage modes
     ///

include/swift/Basic/OptionSet.h

@@ -88,7 +88,7 @@ class OptionSet {
   }
 
   /// Check if this option set contains the exact same options as the given set.
-  bool containsOnly(OptionSet set) {
+  bool containsOnly(OptionSet set) const {
     return Storage == set.Storage;
   }
 

include/swift/Basic/Statistics.def

@@ -187,6 +187,12 @@ FRONTEND_STATISTIC(Parse, NumIterableDeclContextParsed)
 /// Number of conformances that were deserialized by this frontend job.
 FRONTEND_STATISTIC(Sema, NumConformancesDeserialized)
 
+/// Number of pairs of modules we've checked for cross-imports.
+FRONTEND_STATISTIC(Sema, NumCrossImportsChecked)
+
+/// Number of pairs of modules we've actually found cross-imports for.
+FRONTEND_STATISTIC(Sema, NumCrossImportsFound)
+
 /// Number of constraint-solving scopes created in the typechecker, while
 /// solving expression type constraints. A rough proxy for "how much work the
 /// expression typechecker did".

include/swift/ClangImporter/ClangModule.h

@@ -118,6 +118,8 @@ class ClangModuleUnit final : public LoadedFile {
   Optional<clang::ExternalASTSource::ASTSourceDescriptor>
   getASTSourceDescriptor() const;
 
+  virtual StringRef getModuleDefiningPath() const override;
+
   static bool classof(const FileUnit *file) {
     return file->getKind() == FileUnitKind::ClangModule;
   }

include/swift/Option/FrontendOptions.td

@@ -126,6 +126,11 @@ def verify_syntax_tree : Flag<["-"], "verify-syntax-tree">,
 def show_diagnostics_after_fatal : Flag<["-"], "show-diagnostics-after-fatal">,
   HelpText<"Keep emitting subsequent diagnostics after a fatal error">;
 
+def enable_cross_import_overlays : Flag<["-"], "enable-cross-import-overlays">,
+  HelpText<"Automatically import declared cross-import overlays.">;
+def disable_cross_import_overlays : Flag<["-"], "disable-cross-import-overlays">,
+  HelpText<"Do not automatically import declared cross-import overlays.">;
+
 def enable_descriptive_diagnostics : Flag<["-"], "enable-descriptive-diagnostics">,
   HelpText<"Show descriptive diagnostic information, if available.">;
 

include/swift/Serialization/SerializedModuleLoader.h

@@ -393,6 +393,8 @@ class SerializedASTFile final : public LoadedFile {
 
   virtual StringRef getFilename() const override;
 
+  virtual StringRef getModuleDefiningPath() const override;
+
   ClassDecl *getMainClass() const override;
 
   bool hasEntryPoint() const override;

lib/AST/DeclContext.cpp

@@ -287,6 +287,12 @@ DeclContext *DeclContext::getModuleScopeContext() const {
   }
 }
 
+void DeclContext::getSeparatelyImportedOverlays(
+    ModuleDecl *declaring, SmallVectorImpl<ModuleDecl *> &overlays) const {
+  if (auto SF = getParentSourceFile())
+    SF->getSeparatelyImportedOverlays(declaring, overlays);
+}
+
 /// Determine whether the given context is generic at any level.
 bool DeclContext::isGenericContext() const {
   auto dc = this;