Merge pull request #21313 from jrose-apple/secret-imports

Early implementation of "implementation-only imports"

Author: jrose-apple

include/swift/AST/Attr.def

@@ -390,6 +390,9 @@ DECL_ATTR(_private, PrivateImport,
 SIMPLE_DECL_ATTR(_alwaysEmitIntoClient, AlwaysEmitIntoClient,
   OnVar | OnSubscript | OnAbstractFunction | UserInaccessible,
   83)
+SIMPLE_DECL_ATTR(_implementationOnly, ImplementationOnly,
+  OnImport | UserInaccessible,
+  84)
 
 #undef TYPE_ATTR
 #undef DECL_ATTR_ALIAS

include/swift/AST/DiagnosticsSema.def

@@ -765,6 +765,9 @@ ERROR(module_not_testable,Fatal,
 ERROR(module_not_compiled_for_private_import,none,
       "module %0 was not compiled for private import", (Identifier))
 
+ERROR(import_implementation_cannot_be_exported,none,
+      "module %0 cannot be both exported and implementation-only", (Identifier))
+
 
 // Operator decls
 ERROR(ambiguous_operator_decls,none,

include/swift/AST/Module.h

@@ -419,18 +419,23 @@ class ModuleDecl : public DeclContext, public TypeDecl {
          SmallVectorImpl<AbstractFunctionDecl *> &results) const;
 
   /// \sa getImportedModules
-  enum class ImportFilter {
-    All,
-    Public,
-    Private
+  enum class ImportFilterKind {
+    /// Include imports declared with `@_exported`.
+    Public = 1 << 0,
+    /// Include "regular" imports with no special annotation.
+    Private = 1 << 1,
+    /// Include imports declared with `@_implementationOnly`.
+    ImplementationOnly = 1 << 2
   };
+  /// \sa getImportedModules
+  using ImportFilter = OptionSet<ImportFilterKind>;
 
   /// Looks up which modules are imported by this module.
   ///
   /// \p filter controls whether public, private, or any imports are included
   /// in this list.
   void getImportedModules(SmallVectorImpl<ImportedModule> &imports,
-                          ImportFilter filter = ImportFilter::Public) const;
+                          ImportFilter filter = ImportFilterKind::Public) const;
 
   /// Looks up which modules are imported by this module, ignoring any that
   /// won't contain top-level decls.
@@ -758,7 +763,7 @@ class FileUnit : public DeclContext {
   /// \see ModuleDecl::getImportedModulesForLookup
   virtual void getImportedModulesForLookup(
       SmallVectorImpl<ModuleDecl::ImportedModule> &imports) const {
-    return getImportedModules(imports, ModuleDecl::ImportFilter::Public);
+    return getImportedModules(imports, ModuleDecl::ImportFilterKind::Public);
   }
 
   /// Generates the list of libraries needed to link this file, based on its
@@ -891,23 +896,29 @@ class SourceFile final : public FileUnit {
     /// This source file has access to private declarations in the imported
     /// module.
     PrivateImport = 0x4,
+
+    /// The imported module is an implementation detail of this file and should
+    /// not be required to be present if the main module is ever imported
+    /// elsewhere.
+    ///
+    /// Mutually exclusive with Exported.
+    ImplementationOnly = 0x8
   };
 
   /// \see ImportFlags
   using ImportOptions = OptionSet<ImportFlags>;
 
-  typedef std::pair<ImportOptions, StringRef> ImportOptionsAndFilename;
-
   struct ImportedModuleDesc {
     ModuleDecl::ImportedModule module;
     ImportOptions importOptions;
     StringRef filename;
 
-    ImportedModuleDesc(ModuleDecl::ImportedModule module, ImportOptions options)
-        : module(module), importOptions(options) {}
     ImportedModuleDesc(ModuleDecl::ImportedModule module, ImportOptions options,
-                       StringRef filename)
-        : module(module), importOptions(options), filename(filename) {}
+                       StringRef filename = {})
+        : module(module), importOptions(options), filename(filename) {
+      assert(!(importOptions.contains(ImportFlags::Exported) &&
+               importOptions.contains(ImportFlags::ImplementationOnly)));
+    }
   };
 
 private:

include/swift/Basic/OptionSet.h

@@ -87,6 +87,15 @@ class OptionSet {
     return !static_cast<bool>(set - *this);
   }
 
+  /// Check if this option set contains the exact same options as the given set.
+  bool containsOnly(OptionSet set) {
+    return Storage == set.Storage;
+  }
+
+  // '==' and '!=' are deliberately not defined because they provide a pitfall
+  // where someone might use '==' but really want 'contains'. If you actually
+  // want '==' behavior, use 'containsOnly'.
+
   /// Produce the union of two option sets.
   friend OptionSet operator|(OptionSet lhs, OptionSet rhs) {
     return OptionSet(lhs.Storage | rhs.Storage);
@@ -114,7 +123,7 @@ class OptionSet {
     return OptionSet(lhs.Storage & ~rhs.Storage);
   }
 
-  /// Produce the intersection of two option sets.
+  /// Produce the difference of two option sets.
   friend OptionSet &operator-=(OptionSet &lhs, OptionSet rhs) {
     lhs.Storage &= ~rhs.Storage;
     return lhs;

include/swift/Serialization/ModuleFile.h

@@ -135,27 +135,48 @@ class ModuleFile
     const StringRef RawPath;
 
   private:
-    unsigned IsExported : 1;
+    using ImportFilterKind = ModuleDecl::ImportFilterKind;
+    const unsigned RawImportControl : 2;
     const unsigned IsHeader : 1;
     const unsigned IsScoped : 1;
 
-    Dependency(StringRef path, bool isHeader, bool exported, bool isScoped)
-      : RawPath(path), IsExported(exported), IsHeader(isHeader),
-        IsScoped(isScoped) {}
+    static unsigned rawControlFromKind(ImportFilterKind importKind) {
+      return llvm::countTrailingZeros(static_cast<unsigned>(importKind));
+    }
+    ImportFilterKind getImportControl() const {
+      return static_cast<ImportFilterKind>(1 << RawImportControl);
+    }
+
+    Dependency(StringRef path, bool isHeader, ImportFilterKind importControl,
+               bool isScoped)
+      : RawPath(path), RawImportControl(rawControlFromKind(importControl)),
+        IsHeader(isHeader), IsScoped(isScoped) {
+      assert(llvm::countPopulation(static_cast<unsigned>(importControl)) == 1 &&
+             "must be a particular filter option, not a bitset");
+      assert(getImportControl() == importControl && "not enough bits");
+    }
 
   public:
-    Dependency(StringRef path, bool exported, bool isScoped)
-      : Dependency(path, false, exported, isScoped) {}
+    Dependency(StringRef path, ImportFilterKind importControl, bool isScoped)
+      : Dependency(path, false, importControl, isScoped) {}
 
     static Dependency forHeader(StringRef headerPath, bool exported) {
-      return Dependency(headerPath, true, exported, false);
+      auto importControl = exported ? ImportFilterKind::Public
+                                    : ImportFilterKind::Private;
+      return Dependency(headerPath, true, importControl, false);
     }
 
     bool isLoaded() const {
       return Import.second != nullptr;
     }
 
-    bool isExported() const { return IsExported; }
+    bool isExported() const {
+      return getImportControl() == ImportFilterKind::Public;
+    }
+    bool isImplementationOnly() const {
+      return getImportControl() == ImportFilterKind::ImplementationOnly;
+    }
+
     bool isHeader() const { return IsHeader; }
     bool isScoped() const { return IsScoped; }
 
@@ -638,11 +659,26 @@ class ModuleFile
   // Out of line to avoid instantiation OnDiskChainedHashTable here.
   ~ModuleFile();
 
-  /// Associates this module file with an AST module.
-  ///
-  /// Returns any error that occurred during association, including validation
-  /// that the module file is compatible with the module it's being loaded as.
-  Status associateWithFileContext(FileUnit *file, SourceLoc diagLoc);
+  /// Associates this module file with the AST node representing it.
+  ///
+  /// Checks that the file is compatible with the AST module it's being loaded
+  /// into, loads any dependencies needed to understand the module, and updates
+  /// the ASTContext and ClangImporter with search paths and other information
+  /// from the module.
+  ///
+  /// \param file The FileUnit that represents this file's place in the AST.
+  /// \param diagLoc A location used for diagnostics that occur during loading.
+  /// This does not include diagnostics about \e this file failing to load,
+  /// but rather other things that might be imported as part of bringing the
+  /// file into the AST.
+  /// \param treatAsPartialModule If true, processes implementation-only
+  /// information instead of assuming the client won't need it and shouldn't
+  /// see it.
+  ///
+  /// \returns any error that occurred during association, such as being
+  /// compiled for a different OS.
+  Status associateWithFileContext(FileUnit *file, SourceLoc diagLoc,
+                                  bool treatAsPartialModule);
 
   /// Checks whether this module can be used.
   Status getStatus() const {

include/swift/Serialization/ModuleFormat.h

@@ -52,7 +52,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 479; // stored property default arg
+const uint16_t SWIFTMODULE_VERSION_MINOR = 480; // Last change: import control
 
 using DeclIDField = BCFixed<31>;
 
@@ -452,14 +452,26 @@ enum class EnumElementRawValueKind : uint8_t {
   /// TODO: Float, string, char, etc.
 };
 
+using EnumElementRawValueKindField = BCFixed<4>;
+
 // These IDs must \em not be renumbered or reordered without incrementing
 // the module version.
 enum class ResilienceExpansion : uint8_t {
   Minimal = 0,
   Maximal,
 };
 
-using EnumElementRawValueKindField = BCFixed<4>;
+// These IDs must \em not be renumbered or reordered without incrementing
+// the module version.
+enum class ImportControl : uint8_t {
+  /// `import FooKit`
+  Normal = 0,
+  /// `@_exported import FooKit`
+  Exported,
+  /// `@_uncheckedImplementationOnly import FooKit`
+  ImplementationOnly
+};
+using ImportControlField = BCFixed<2>;
 
 /// The various types of blocks that can occur within a serialized Swift
 /// module.
@@ -636,8 +648,8 @@ namespace input_block {
 
   using ImportedModuleLayout = BCRecordLayout<
     IMPORTED_MODULE,
-    BCFixed<1>, // exported?
-    BCFixed<1>, // scoped?
+    ImportControlField, // import kind
+    BCFixed<1>,         // scoped?
     BCBlob // module name, with submodule path pieces separated by \0s.
            // If the 'scoped' flag is set, the final path piece is an access
            // path within the module.

include/swift/Serialization/SerializedModuleLoader.h

@@ -94,7 +94,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   FileUnit *loadAST(ModuleDecl &M, Optional<SourceLoc> diagLoc,
                     std::unique_ptr<llvm::MemoryBuffer> moduleInputBuffer,
                     std::unique_ptr<llvm::MemoryBuffer> moduleDocInputBuffer,
-                    bool isFramework = false);
+                    bool isFramework, bool treatAsPartialModule);
 
   /// Check whether the module with a given name can be imported without
   /// importing it.

lib/AST/Module.cpp

@@ -1020,21 +1020,18 @@ void
 SourceFile::getImportedModules(SmallVectorImpl<ModuleDecl::ImportedModule> &modules,
                                ModuleDecl::ImportFilter filter) const {
   assert(ASTStage >= Parsed || Kind == SourceFileKind::SIL);
+  assert(filter && "no imports requested?");
   for (auto desc : Imports) {
-    switch (filter) {
-    case ModuleDecl::ImportFilter::All:
-      break;
-    case ModuleDecl::ImportFilter::Public:
-      if (!desc.importOptions.contains(ImportFlags::Exported))
-        continue;
-      break;
-    case ModuleDecl::ImportFilter::Private:
-      if (desc.importOptions.contains(ImportFlags::Exported))
-        continue;
-      break;
-    }
+    ModuleDecl::ImportFilterKind requiredKind;
+    if (desc.importOptions.contains(ImportFlags::Exported))
+      requiredKind = ModuleDecl::ImportFilterKind::Public;
+    else if (desc.importOptions.contains(ImportFlags::ImplementationOnly))
+      requiredKind = ModuleDecl::ImportFilterKind::ImplementationOnly;
+    else
+      requiredKind = ModuleDecl::ImportFilterKind::Private;
 
-    modules.push_back(desc.module);
+    if (filter.contains(requiredKind))
+      modules.push_back(desc.module);
   }
 }
 
@@ -1277,9 +1274,14 @@ forAllImportedModules(ModuleDecl *topLevel, ModuleDecl::AccessPathTy thisPath,
   llvm::SmallSet<ImportedModule, 32, ModuleDecl::OrderImportedModules> visited;
   SmallVector<ImportedModule, 32> stack;
 
-  auto filter = respectVisibility ? ModuleDecl::ImportFilter::Public
-                                  : ModuleDecl::ImportFilter::All;
-  topLevel->getImportedModules(stack, filter);
+  ModuleDecl::ImportFilter filter = ModuleDecl::ImportFilterKind::Public;
+  if (!respectVisibility)
+    filter |= ModuleDecl::ImportFilterKind::Private;
+
+  ModuleDecl::ImportFilter topLevelFilter = filter;
+  if (!respectVisibility)
+    topLevelFilter |= ModuleDecl::ImportFilterKind::ImplementationOnly;
+  topLevel->getImportedModules(stack, topLevelFilter);
 
   // Make sure the top-level module is first; we want pre-order-ish traversal.
   AccessPathTy overridingPath;
@@ -1330,9 +1332,11 @@ bool FileUnit::forAllVisibleModules(
 
   if (auto SF = dyn_cast<SourceFile>(this)) {
     // Handle privately visible modules as well.
-    // FIXME: Should this apply to all FileUnits?
+    ModuleDecl::ImportFilter importFilter;
+    importFilter |= ModuleDecl::ImportFilterKind::Private;
+    importFilter |= ModuleDecl::ImportFilterKind::ImplementationOnly;
     SmallVector<ModuleDecl::ImportedModule, 4> imports;
-    SF->getImportedModules(imports, ModuleDecl::ImportFilter::Private);
+    SF->getImportedModules(imports, importFilter);
     for (auto importPair : imports)
       if (!importPair.second->forAllVisibleModules(importPair.first, fn))
         return false;

lib/AST/UnqualifiedLookup.cpp

@@ -1044,8 +1044,12 @@ void UnqualifiedLookupFactory::recordDependencyOnTopLevelName(
 void UnqualifiedLookupFactory::addImportedResults(DeclContext *const dc) {
   // Add private imports to the extra search list.
   SmallVector<ModuleDecl::ImportedModule, 8> extraImports;
-  if (auto FU = dyn_cast<FileUnit>(dc))
-    FU->getImportedModules(extraImports, ModuleDecl::ImportFilter::Private);
+  if (auto FU = dyn_cast<FileUnit>(dc)) {
+    ModuleDecl::ImportFilter importFilter;
+    importFilter |= ModuleDecl::ImportFilterKind::Private;
+    importFilter |= ModuleDecl::ImportFilterKind::ImplementationOnly;
+    FU->getImportedModules(extraImports, importFilter);
+  }
 
   using namespace namelookup;
   SmallVector<ValueDecl *, 8> CurModuleResults;