Experimental: Extend ClangImporter to import clang modules from DWARF

When debugging Objective-C or C++ code on Darwin, the debug info
collected by dsymutil in the .dSYM bundle is entirely
self-contained. It is possible to debug a program, set breakpoints and
print variables even without having the complete original source code
or a matching SDK available. With Swift, this is currently not the
case. Even though .dSYM bundles contain the binary .swiftmodule for
all Swift modules, any Clang modules that the Swift modules depend on,
still need to be imported from source to even get basic LLDB
functionality to work. If ClangImporter fails to import a Clang
module, effectively the entire Swift module depending on it gets
poisoned.

This patch is addressing this issue by introducing a ModuleLoader that
can ask queries about Clang Decls to LLDB, since LLDB knows how to
reconstruct Clang decls from DWARF and clang -gmodules producxes full
debug info for Clang modules that is embedded into the .dSYM budle.

This initial version does not contain any advanced functionality at
all, it merely produces an empty ModuleDecl. Intertestingly, even this
is a considerable improvement over the status quo. LLDB can now print
Swift-only variables in modules with failing Clang depenecies, and
becuase of fallback mechanisms that were implemented earlier, it can
even display the contents of pure Objective-C objects that are
imported into Swift. C structs obviously don't work yet.

rdar://problem/36032653

Author: adrian-prantl

include/swift/AST/Module.h

@@ -94,6 +94,8 @@ enum class FileUnitKind {
   SerializedAST,
   /// An imported Clang module.
   ClangModule,
+  /// A Clang module imported from DWARF.
+  DWARFModule
 };
 
 enum class SourceFileKind {
@@ -1326,7 +1328,8 @@ class LoadedFile : public FileUnit {
 
   static bool classof(const FileUnit *file) {
     return file->getKind() == FileUnitKind::SerializedAST ||
-           file->getKind() == FileUnitKind::ClangModule;
+           file->getKind() == FileUnitKind::ClangModule ||
+           file->getKind() == FileUnitKind::DWARFModule;
   }
   static bool classof(const DeclContext *DC) {
     return isa<FileUnit>(DC) && classof(cast<FileUnit>(DC));

include/swift/Basic/LangOptions.h

@@ -98,9 +98,12 @@ namespace swift {
     /// Support for alternate usage modes
     ///
 
-    /// \brief Enable features useful for running in the debugger.
+    /// Enable features useful for running in the debugger.
     bool DebuggerSupport = false;
 
+    /// Enable the DWARFImporter. Only used by lldb-moduleimport-test.
+    bool EnableDWARFImporter = false;
+    
     /// Allows using identifiers with a leading dollar.
     bool EnableDollarIdentifiers = false;
 

include/swift/DWARFImporter/DWARFImporter.h

@@ -0,0 +1,101 @@
+//===--- DWARFImporter.h - Import Clang Modules -----------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// \file This file implements support for loading Clang modules that were
+//       reconstructed from DWARF into Swift.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SWIFT_DWARF_IMPORTER_H
+#define SWIFT_DWARF_IMPORTER_H
+
+#include "swift/AST/ClangModuleLoader.h"
+
+namespace llvm {
+}
+
+namespace clang {
+}
+
+namespace swift {
+
+/// Class that imports Clang modules into Swift, mapping directly
+/// from Clang ASTs over to Swift ASTs.
+class DWARFImporter final : public ClangModuleLoader {
+  friend class ClangModuleUnit;
+
+public:
+  class Implementation;
+
+private:
+  Implementation &Impl;
+
+  DWARFImporter(ASTContext &ctx, const ClangImporterOptions &clangImporterOpts,
+                DependencyTracker *tracker);
+
+public:
+  /// Create a new DWARF importer that can import a Clang Modules from DWARF
+  /// into the given ASTContext.
+  ///
+  /// \param ctx The ASTContext into which the module will be imported.
+  /// The ASTContext's SearchPathOptions will be used for the DWARF importer.
+  ///
+  /// \param importerOpts The options to use for the DWARF importer.
+  ///
+  /// \param tracker The object tracking files this compilation depends on.
+  ///
+  /// \returns a new DWARF module importer, or null (with a diagnostic) if
+  /// an error occurred.
+  static std::unique_ptr<DWARFImporter>
+  create(ASTContext &ctx,
+         const ClangImporterOptions &importerOpts,
+         DependencyTracker *tracker = nullptr);
+
+  DWARFImporter(const DWARFImporter &) = delete;
+  DWARFImporter(DWARFImporter &&) = delete;
+  DWARFImporter &operator=(const DWARFImporter &) = delete;
+  DWARFImporter &operator=(DWARFImporter &&) = delete;
+
+  ~DWARFImporter();
+
+  /// Check whether the module with a given name can be imported without
+  /// importing it.
+  ///
+  /// Note that even if this check succeeds, errors may still occur if the
+  /// module is loaded in full.
+  bool canImportModule(std::pair<Identifier, SourceLoc> named) override;
+  bool addSearchPath(StringRef newSearchPath, bool isFramework,
+                     bool isSystem) override;
+  ModuleDecl *
+  loadModule(SourceLoc importLoc,
+             ArrayRef<std::pair<Identifier, SourceLoc>> path) override;
+  bool
+  isInOverlayModuleForImportedModule(const DeclContext *overlayDC,
+                                     const DeclContext *importedDC) override;
+  void loadExtensions(NominalTypeDecl *nominal,
+                      unsigned previousGeneration) override;
+  void loadObjCMethods(
+      ClassDecl *classDecl, ObjCSelector selector, bool isInstanceMethod,
+      unsigned previousGeneration,
+      llvm::TinyPtrVector<AbstractFunctionDecl *> &methods) override;
+  ModuleDecl *getImportedHeaderModule() const override;
+  void verifyAllModules() override;
+  clang::ASTContext &getClangASTContext() const override;
+  clang::Preprocessor &getClangPreprocessor() const override;
+  clang::Sema &getClangSema() const override;
+  const clang::CompilerInstance &getClangInstance() const override;
+
+  void printStatistics() const override;
+};
+
+} // end namespace swift
+
+#endif

lib/AST/ASTContext.cpp

@@ -1380,6 +1380,7 @@ void ASTContext::addExternalDecl(Decl *decl) {
 void ASTContext::addSynthesizedDecl(Decl *decl) {
   auto *mod = cast<FileUnit>(decl->getDeclContext()->getModuleScopeContext());
   if (mod->getKind() == FileUnitKind::ClangModule ||
+      mod->getKind() == FileUnitKind::DWARFModule ||
       mod->getKind() == FileUnitKind::SerializedAST) {
     ExternalDefinitions.insert(decl);
     return;
@@ -1422,11 +1423,10 @@ void ASTContext::addSearchPath(StringRef searchPath, bool isFramework,
 
 void ASTContext::addModuleLoader(std::unique_ptr<ModuleLoader> loader,
                                  bool IsClang) {
-  if (IsClang) {
-    assert(!getImpl().TheClangModuleLoader && "Already have a Clang module loader");
+  if (IsClang && !getImpl().TheClangModuleLoader)
     getImpl().TheClangModuleLoader =
-      static_cast<ClangModuleLoader *>(loader.get());
-  }
+        static_cast<ClangModuleLoader *>(loader.get());
+
   getImpl().ModuleLoaders.push_back(std::move(loader));
 }
 

lib/AST/ASTMangler.cpp

@@ -1309,7 +1309,8 @@ ASTMangler::getSpecialManglingContext(const ValueDecl *decl) {
   // Clang decls. Check getKind() directly to avoid a layering dependency.
   //   known-context ::= 'SC'
   if (auto file = dyn_cast<FileUnit>(decl->getDeclContext())) {
-    if (file->getKind() == FileUnitKind::ClangModule) {
+    if (file->getKind() == FileUnitKind::ClangModule ||
+        file->getKind() == FileUnitKind::DWARFModule) {
       if (decl->getClangDecl())
         return ASTMangler::ObjCContext;
       return ASTMangler::ClangImporterContext;
@@ -2273,7 +2274,8 @@ ASTMangler::appendProtocolConformance(const ProtocolConformance *conformance) {
 
     bool needsModule = true;
     if (auto *file = dyn_cast<FileUnit>(topLevelContext)) {
-      if (file->getKind() == FileUnitKind::ClangModule) {
+      if (file->getKind() == FileUnitKind::ClangModule ||
+          file->getKind() == FileUnitKind::DWARFModule) {
         if (conformance->getProtocol()->hasClangNode())
           appendOperator("So");
         else

lib/AST/ASTPrinter.cpp

@@ -3354,7 +3354,8 @@ class TypePrinter : public TypeVisitor<TypePrinter> {
 
     // Don't print qualifiers for imported types.
     for (auto File : M->getFiles()) {
-      if (File->getKind() == FileUnitKind::ClangModule)
+      if (File->getKind() == FileUnitKind::ClangModule ||
+          File->getKind() == FileUnitKind::DWARFModule)
         return false;
     }
 

lib/AST/DeclContext.cpp

@@ -537,6 +537,7 @@ unsigned DeclContext::printContext(raw_ostream &OS, unsigned indent) const {
       break;
     case FileUnitKind::SerializedAST:
     case FileUnitKind::ClangModule:
+    case FileUnitKind::DWARFModule:
       OS << " file=\"" << cast<LoadedFile>(this)->getFilename() << "\"";
       break;
     }

lib/AST/Module.cpp

@@ -876,6 +876,7 @@ lookupOperatorDeclForName(const FileUnit &File, SourceLoc Loc, Identifier Name,
     break;
   case FileUnitKind::SerializedAST:
   case FileUnitKind::ClangModule:
+  case FileUnitKind::DWARFModule:
     return OperatorLookup<OP_DECL>::lookup(cast<LoadedFile>(File), Name);
   }
 

lib/AST/ProtocolConformance.cpp

@@ -1297,7 +1297,8 @@ void NominalTypeDecl::prepareConformanceTable() const {
   // via the Clang importer, don't add any synthesized conformances.
   auto *file = cast<FileUnit>(getModuleScopeContext());
   if (file->getKind() != FileUnitKind::Source &&
-      file->getKind() != FileUnitKind::ClangModule) {
+      file->getKind() != FileUnitKind::ClangModule &&
+      file->getKind() != FileUnitKind::DWARFModule) {
     return;
   }
 

lib/CMakeLists.txt

@@ -19,6 +19,7 @@ add_subdirectory(Basic)
 add_subdirectory(ClangImporter)
 add_subdirectory(Demangling)
 add_subdirectory(Driver)
+add_subdirectory(DWARFImporter)
 add_subdirectory(Frontend)
 add_subdirectory(FrontendTool)
 add_subdirectory(Index)

lib/DWARFImporter/CMakeLists.txt

@@ -0,0 +1,8 @@
+add_swift_host_library(swiftDWARFImporter STATIC
+  DWARFImporter.cpp
+  LINK_LIBRARIES
+    swiftAST
+    swiftParse
+)
+
+#add_dependencies(swiftDWARFImporter )