[cxx-interop] Support compiling using a custom C++ stdlib

Author: hyp

include/swift/AST/Decl.h

@@ -695,7 +695,7 @@ class alignas(1 << DeclAlignInBits) Decl : public ASTAllocated<Decl> {
     HasAnyUnavailableDuringLoweringValues : 1
   );
 
-  SWIFT_INLINE_BITFIELD(ModuleDecl, TypeDecl, 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1,
+  SWIFT_INLINE_BITFIELD(ModuleDecl, TypeDecl, 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1,
     /// If the module is compiled as static library.
     StaticLibrary : 1,
 
@@ -754,7 +754,12 @@ class alignas(1 << DeclAlignInBits) Decl : public ASTAllocated<Decl> {
     HasCxxInteroperability : 1,
 
     /// Whether this module has been built with -experimental-allow-non-resilient-access.
-    AllowNonResilientAccess : 1
+    AllowNonResilientAccess : 1,
+
+    /// Whether this module has been built using a sealed C++ interoperability configuration
+    /// that's broadly incompatible with default C++ interoperability support. For example, this is
+    /// set for modules that use a custom libc++ with C++ interoperability.
+    HasSealedCxxInteroperability : 1
   );
 
   SWIFT_INLINE_BITFIELD(PrecedenceGroupDecl, Decl, 1+2,

include/swift/AST/DiagnosticsClangImporter.def

@@ -129,6 +129,8 @@ WARNING(libstdcxx_modulemap_not_found, none,
         "module map for libstdc++ not found for '%0'; C++ stdlib may be unavailable",
         (StringRef))
 
+ERROR(libcxx_custom_prohibits_system_cxxstdlib_module, none, "transitive import of system's C++ stdlib module '%0' prohibited in custom C++ stdlib mode", (StringRef))
+
 WARNING(api_pattern_attr_ignored, none,
         "'%0' swift attribute ignored on type '%1': type is not copyable or destructible",
         (StringRef, StringRef))

include/swift/AST/DiagnosticsSema.def

@@ -931,6 +931,10 @@ ERROR(need_cxx_interop_to_import_module,none,
       (Identifier))
 NOTE(enable_cxx_interop_docs,none,
      "visit https://www.swift.org/documentation/cxx-interop/project-build-setup to learn how to enable C++ interoperability", ())
+ERROR(need_custom_cxx_stdlib_to_import_module,none,
+      "module %0 uses system's C++ standard library, but "
+      "current compilation uses a custom C++ standard library",
+      (Identifier))
 
 ERROR(modularization_issue_decl_moved,Fatal,
       "reference to %select{top-level declaration|type}0 %1 broken by a context change; "

include/swift/AST/Module.h

@@ -689,6 +689,16 @@ class ModuleDecl
     Bits.ModuleDecl.HasCxxInteroperability = enabled;
   }
 
+  /// Returns true if this module was built with sealed C++ interoperability
+  /// configuration, that's broadly incompatible with default C++
+  /// interoperability support.
+  bool hasSealedCxxInteroperability() const {
+    return Bits.ModuleDecl.HasSealedCxxInteroperability;
+  }
+  void setHasSealedCxxInteroperability(bool enabled = true) {
+    Bits.ModuleDecl.HasSealedCxxInteroperability = enabled;
+  }
+
   /// \returns true if this module is a system module; note that the StdLib is
   /// considered a system module.
   bool isSystemModule() const {

include/swift/Basic/LangOptions.h

@@ -328,6 +328,10 @@ namespace swift {
     /// when importing Swift modules that enable C++ interoperability.
     bool RequireCxxInteropToImportCxxInteropModule = true;
 
+    /// Use a custom libc++ at the specified path when importing
+    // and building Clang modules with C++ interoperability enabled.
+    std::string cxxInteropCustomLibcxxPath;
+
     /// On Darwin platforms, use the pre-stable ABI's mark bit for Swift
     /// classes instead of the stable ABI's bit. This is needed when
     /// targeting OSes prior to macOS 10.14.4 and iOS 12.2, where
@@ -763,6 +767,12 @@ namespace swift {
       return hashValue;
     }
 
+    /// Return true when using a custom C++ standard library for imported
+    /// C and C++ modules.
+    bool isUsingCustomCxxStdLib() const {
+      return !cxxInteropCustomLibcxxPath.empty();
+    }
+
   private:
     llvm::SmallVector<std::string, 2> AtomicBitWidths;
     llvm::SmallVector<std::pair<PlatformConditionKind, std::string>, 10>

include/swift/Option/FrontendOptions.td

@@ -939,6 +939,11 @@ def cxx_interop_disable_requirement_at_import :
   HelpText<"Do not require C++ interoperability to be enabled when importing a Swift module that enables C++ interoperability">,
   Flags<[FrontendOption, HelpHidden]>;
 
+def cxx_interop_libcxx_path: JoinedOrSeparate<["-"], "clang-libcxx-path">,
+  HelpText<"Use a custom libc++ as the C++ standard library when importing Clang modules">,
+  MetaVarName<"<path>">,
+  Flags<[FrontendOption, HelpHidden]>;
+
 def use_malloc : Flag<["-"], "use-malloc">,
   HelpText<"Allocate internal data structures using malloc "
            "(for memory debugging)">;

include/swift/Serialization/Validation.h

@@ -141,6 +141,7 @@ class ExtendedValidationInfo {
     unsigned IsConcurrencyChecked : 1;
     unsigned HasCxxInteroperability : 1;
     unsigned AllowNonResilientAccess: 1;
+    unsigned HasSealedCxxInteroperability : 1;
   } Bits;
 public:
   ExtendedValidationInfo() : Bits() {}
@@ -235,6 +236,12 @@ class ExtendedValidationInfo {
   void setHasCxxInteroperability(bool val) {
     Bits.HasCxxInteroperability = val;
   }
+  bool hasSealedCxxInteroperability() const {
+    return Bits.HasSealedCxxInteroperability;
+  }
+  void setHasSealedCxxInteroperability(bool val) {
+    Bits.HasSealedCxxInteroperability = val;
+  }
 };
 
 struct SearchPath {

lib/AST/Module.cpp

@@ -723,6 +723,7 @@ ModuleDecl::ModuleDecl(Identifier name, ASTContext &ctx,
   Bits.ModuleDecl.ObjCNameLookupCachePopulated = 0;
   Bits.ModuleDecl.HasCxxInteroperability = 0;
   Bits.ModuleDecl.AllowNonResilientAccess = 0;
+  Bits.ModuleDecl.HasSealedCxxInteroperability = 0;
 }
 
 void ModuleDecl::setIsSystemModule(bool flag) {

lib/ClangImporter/ClangImporter.cpp

@@ -506,6 +506,24 @@ void importer::getNormalInvocationArguments(
       "-isystem", searchPathOpts.RuntimeResourcePath,
   });
 
+  if (ctx.LangOpts.isUsingCustomCxxStdLib()) {
+    // Add a define that ensures Clang differentiates between imported Clang
+    // modules with a
+    // custom libc++ path and regular Clang modules that use system's C++
+    // standard library.
+    invocationArgStrs.insert(invocationArgStrs.end(),
+                             {"-D__swift_use_custom_libcxx__"});
+
+    // Add a placeholder custom C++ stdlib directory before other system include
+    // directories. The importer will substitute the actual directory value
+    // after preloading any modules that are incompatible with the custom C++
+    // stdlib. It's important to put it before other system include directories
+    // as even with -nostdinc++, platforms like Windows might still keep the
+    // search paths for system's C++ headers active in the Clang instance.
+    invocationArgStrs.insert(invocationArgStrs.end(),
+                             {"-isystem<placeholder-custom-cxx-stdlib-dir>"});
+  }
+
   // Enable Position Independence.  `-fPIC` is not supported on Windows, which
   // is implicitly position independent.
   if (!triple.isOSWindows())
@@ -897,6 +915,12 @@ importer::addCommonInvocationArguments(
     invocationArgStrs.push_back("-fbuild-session-file=" + importerOpts.BuildSessionFilePath);
   }
 
+  if (ctx.LangOpts.isUsingCustomCxxStdLib()) {
+    // Do not add system's C++ standard library include paths when using
+    // a custom C++ standard libary.
+    invocationArgStrs.push_back("-nostdinc++");
+  }
+
   for (auto extraArg : importerOpts.ExtraArgs) {
     invocationArgStrs.push_back(extraArg);
   }
@@ -1297,11 +1321,29 @@ ClangImporter::create(ASTContext &ctx,
       return nullptr;
   }
 
+  llvm::SmallVector<StringRef, 2> extraPreImports;
+  // Prebuild 'SwiftShims' before using a custom C++ stdlib, to
+  // ensure that SwiftShims won't depend on the custom C++ stdlib.
+  if (ctx.LangOpts.isUsingCustomCxxStdLib()) {
+    extraPreImports.push_back("SwiftShims");
+    // On Windows, prebuild 'ucrt' before using a custom C++ stdlib,
+    // to ensure there's no circular dependencies between the two
+    // modules.
+    if (ctx.LangOpts.Target.isOSWindows())
+      extraPreImports.push_back("ucrt");
+  }
+
   {
     // Create an almost-empty memory buffer.
-    auto sourceBuffer = llvm::MemoryBuffer::getMemBuffer(
-      "extern int __swift __attribute__((unavailable));",
-      Implementation::moduleImportBufferName);
+    SmallString<128> defaultBuffer(
+        "extern int __swift __attribute__((unavailable));");
+    for (const auto &import : extraPreImports) {
+      defaultBuffer.append("\n#pragma clang module import ");
+      defaultBuffer.append(import);
+      defaultBuffer.push_back('\n');
+    }
+    auto sourceBuffer = llvm::MemoryBuffer::getMemBufferCopy(
+        defaultBuffer, Implementation::moduleImportBufferName);
     clang::PreprocessorOptions &ppOpts =
         importer->Impl.Invocation->getPreprocessorOpts();
     ppOpts.addRemappedFile(Implementation::moduleImportBufferName,
@@ -1481,6 +1523,34 @@ ClangImporter::create(ASTContext &ctx,
   auto *CB = new HeaderImportCallbacks(importer->Impl);
   clangPP.addPPCallbacks(std::unique_ptr<clang::PPCallbacks>(CB));
 
+  if (ctx.LangOpts.isUsingCustomCxxStdLib()) {
+    // Adjust the header search options to point to the custom C++ stdlib
+    // include directory instead of the placeholder custom C++ stdlib, after
+    // making sure some incompatible requirements are prebuilt already.
+    bool hasAdjusted = false;
+    for (auto &E : instance.getHeaderSearchOpts().UserEntries) {
+      if (E.Path == "<placeholder-custom-cxx-stdlib-dir>") {
+        E.Path = ctx.LangOpts.cxxInteropCustomLibcxxPath;
+        hasAdjusted = true;
+      }
+    }
+    (void)hasAdjusted; // in case of no asserts.
+    assert(hasAdjusted && "unable to set custom c++ stdlib include path");
+    // Recreate the header search info for the current instance to match
+    // the updated header search options.
+    clang::ApplyHeaderSearchOptions(
+        clangPP.getHeaderSearchInfo(), instance.getHeaderSearchOpts(),
+        instance.getLangOpts(), instance.getTarget().getTriple());
+
+    if (importerOpts.DumpClangDiagnostics) {
+      llvm::errs()
+          << "Adjusted include paths to account for custom C++ stdlib:";
+      for (auto &E : instance.getHeaderSearchOpts().UserEntries)
+        llvm::errs() << " '" << E.Path << "'";
+      llvm::errs() << "\n";
+    }
+  }
+
   // Create the selectors we'll be looking for.
   auto &clangContext = importer->Impl.Instance->getASTContext();
   importer->Impl.objectAtIndexedSubscript
@@ -2274,6 +2344,10 @@ ModuleDecl *ClangImporter::Implementation::loadModule(
       path.front().Item.str().starts_with("std_"))
     return nullptr;
   if (path.front().Item == ctx.Id_CxxStdlib) {
+    // The 'CxxStdlib' module is not importable when using a custom
+    // C++ standard libary.
+    if (ctx.LangOpts.isUsingCustomCxxStdLib())
+      return nullptr;
     ImportPath::Builder adjustedPath(ctx.getIdentifier("std"), importLoc);
     adjustedPath.append(path.getSubmodulePath());
     path = adjustedPath.copyTo(ctx).getModulePath(ImportKind::Module);
@@ -2640,6 +2714,27 @@ ClangModuleUnit *ClangImporter::Implementation::getWrapperForModule(
   if (ClangModuleUnit *cached = cacheEntry.getPointer())
     return cached;
 
+  // Ensure system's C++ stdlib modules aren't imported when
+  // using a custom C++ standard library. This is especially important for
+  // Windows with MSVC STL, as the MSVC C++ module is located in a directory
+  // shared with other system modules, and thus could be still found using
+  // Clang's header search. However, MSVC's 'std_config' module is except from
+  // this check as it contains some common config macros that are needed by
+  // other parts of the SDK.
+  if (SwiftContext.LangOpts.isUsingCustomCxxStdLib() &&
+      isCxxStdModule(underlying) &&
+      !(SwiftContext.LangOpts.Target.isOSWindows() &&
+        underlying->Name == "std_config")) {
+    auto maybeCustomDir = Instance->getFileManager().getOptionalDirectoryRef(
+        SwiftContext.LangOpts.cxxInteropCustomLibcxxPath);
+    if (underlying->Directory.has_value() && maybeCustomDir &&
+        !underlying->Directory->isSameRef(*maybeCustomDir)) {
+      diagnose(SourceLoc(),
+               diag::libcxx_custom_prohibits_system_cxxstdlib_module,
+               underlying->Name);
+    }
+  }
+
   // FIXME: Handle hierarchical names better.
   Identifier name = underlying->Name == "std"
                         ? SwiftContext.Id_CxxStdlib

lib/Frontend/CompilerInvocation.cpp

@@ -1241,6 +1241,9 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.CxxInteropGettersSettersAsProperties = Args.hasArg(OPT_cxx_interop_getters_setters_as_properties);
   Opts.RequireCxxInteropToImportCxxInteropModule =
       !Args.hasArg(OPT_cxx_interop_disable_requirement_at_import);
+  if (const auto *A = Args.getLastArg(OPT_cxx_interop_libcxx_path)) {
+    Opts.cxxInteropCustomLibcxxPath = A->getValue();
+  }
 
   Opts.VerifyAllSubstitutionMaps |= Args.hasArg(OPT_verify_all_substitution_maps);
 

lib/Frontend/Frontend.cpp

@@ -1397,8 +1397,19 @@ ModuleDecl *CompilerInstance::getMainModule() const {
     if (Invocation.getLangOptions().isSwiftVersionAtLeast(6))
       MainModule->setIsConcurrencyChecked(true);
     if (Invocation.getLangOptions().EnableCXXInterop &&
-        Invocation.getLangOptions().RequireCxxInteropToImportCxxInteropModule)
-      MainModule->setHasCxxInteroperability();
+        Invocation.getLangOptions().RequireCxxInteropToImportCxxInteropModule) {
+      // By default, mark this module as using C++ interoperability, which
+      // will require users to also turn on C++ interoperability.
+      if (Invocation.getLangOptions().RequireCxxInteropToImportCxxInteropModule)
+        MainModule->setHasCxxInteroperability();
+      // Using C++ interoperability with a custom C++ stdlib enforces
+      // a best-effort seal that prevents the use of C++ types accross module
+      // boundaries, while ensuring that type layout from such module will
+      // not be constructed incorrectly without correct C++ libraries when used
+      // accross a non-resilient module boundary.
+      if (Invocation.getLangOptions().isUsingCustomCxxStdLib())
+        MainModule->setHasSealedCxxInteroperability();
+    }
     if (Invocation.getLangOptions().AllowNonResilientAccess)
       MainModule->setAllowNonResilientAccess();
 

lib/IRGen/GenDecl.cpp

@@ -487,10 +487,13 @@ void IRGenModule::emitSourceFile(SourceFile &SF) {
   // (std) if available.
   if (Context.LangOpts.EnableCXXInterop) {
     const llvm::Triple &target = Context.LangOpts.Target;
-    if (target.isOSDarwin())
-      this->addLinkLibrary(LinkLibrary("c++", LibraryKind::Library));
-    else if (target.isOSLinux())
-      this->addLinkLibrary(LinkLibrary("stdc++", LibraryKind::Library));
+    bool useCustomCxxLib = Context.LangOpts.isUsingCustomCxxStdLib();
+    if (!useCustomCxxLib) {
+      if (target.isOSDarwin())
+        this->addLinkLibrary(LinkLibrary("c++", LibraryKind::Library));
+      else if (target.isOSLinux())
+        this->addLinkLibrary(LinkLibrary("stdc++", LibraryKind::Library));
+    }
 
     // Do not try to link Cxx with itself.
     if (!getSwiftModule()->getName().is("Cxx")) {
@@ -504,11 +507,13 @@ void IRGenModule::emitSourceFile(SourceFile &SF) {
     }
 
     // Do not try to link CxxStdlib with the C++ standard library, Cxx or
-    // itself.
+    // itself. Also, do not link CxxStdlib when using a custom C++
+    // standard library.
     if (llvm::none_of(llvm::ArrayRef{"Cxx", "CxxStdlib", "std"},
                       [M = getSwiftModule()->getName().str()](StringRef Name) {
                         return M == Name;
-                      })) {
+                      }) &&
+        !useCustomCxxLib) {
       // Only link with CxxStdlib on platforms where the overlay is available.
       switch (target.getOS()) {
       case llvm::Triple::Linux:

lib/Sema/TypeCheckAccess.cpp

@@ -1989,9 +1989,11 @@ swift::getDisallowedOriginKind(const Decl *decl,
   }
 
   // C++ APIs do not support library evolution.
+  // FIXME: switch to seal?
   if (SF->getASTContext().LangOpts.EnableCXXInterop && where.getDeclContext() &&
       where.getDeclContext()->getAsDecl() &&
-      where.getDeclContext()->getAsDecl()->getModuleContext()->isResilient() &&
+      (where.getDeclContext()->getAsDecl()->getModuleContext()->isResilient() ||
+       !SF->getASTContext().LangOpts.cxxInteropCustomLibcxxPath.empty()) &&
       decl->hasClangNode() && !decl->getModuleContext()->isSwiftShimsModule() &&
       isFragileClangNode(decl->getClangNode()))
     return DisallowedOriginKind::FragileCxxAPI;

lib/Serialization/Deserialization.cpp

@@ -8102,13 +8102,18 @@ void ModuleFile::loadAllMembers(Decl *container, uint64_t contextData) {
 
   SmallVector<Decl *, 16> members;
   members.reserve(rawMemberIDs.size());
+  bool hasSealed =
+      container->getModuleContext()->hasSealedCxxInteroperability();
+  bool reportDeserializationErrors =
+      (hasSealed && !isa<ClassDecl>(container)) ||
+      !getContext().LangOpts.EnableDeserializationRecovery;
   for (DeclID rawID : rawMemberIDs) {
     Expected<Decl *> next = getDeclChecked(rawID);
     if (next) {
       assert(next.get() && "unchecked error deserializing next member");
       members.push_back(next.get());
     } else {
-      if (!getContext().LangOpts.EnableDeserializationRecovery)
+      if (reportDeserializationErrors)
         fatal(next.takeError());
 
       Decl *suppliedMissingMember = handleErrorAndSupplyMissingMember(

lib/Serialization/ModuleFile.h

@@ -646,6 +646,12 @@ class ModuleFile
     return Core->Bits.HasCxxInteroperability;
   }
 
+  /// Whether this module was built with sealed C++ interoperability enabled,
+  /// that's broadly incompatible with default C++ interoperability support.
+  bool hasSealedCxxInteroperability() const {
+    return Core->Bits.HasSealedCxxInteroperability;
+  }
+
   /// Whether the module is resilient. ('-enable-library-evolution')
   ResilienceStrategy getResilienceStrategy() const {
     return ResilienceStrategy(Core->Bits.ResilienceStrategy);