Merge pull request #7259 from adrian-prantl/107031048-5.9

Parse plugin search paths from swift modules that are interactively

Author: adrian-prantl

lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp

@@ -1085,7 +1085,7 @@ static void printASTValidationError(
     llvm::raw_ostream &errs,
     const swift::serialization::ValidationInfo &ast_info,
     const swift::serialization::ExtendedValidationInfo &ext_ast_info,
-    Module &module, StringRef module_buf, bool invalid_name,
+    StringRef module_name, StringRef module_buf, bool invalid_name,
     bool invalid_size) {
   const char *error = getImportFailureString(ast_info.status);
   errs << "AST validation error";
@@ -1110,9 +1110,9 @@ static void printASTValidationError(
   - SDK path: {7}
   - Clang Importer Options:
 )",
-           ast_info.name, module.GetSpecificationDescription(), error,
-           ast_info.targetTriple, ast_info.shortVersion, ast_info.bytes,
-           module_buf.size(), ext_ast_info.getSDKPath());
+           ast_info.name, module_name, error, ast_info.targetTriple,
+           ast_info.shortVersion, ast_info.bytes, module_buf.size(),
+           ext_ast_info.getSDKPath());
   for (StringRef ExtraOpt : ext_ast_info.getExtraClangImporterOptions())
     LLDB_LOG(log, "  -- {0}", ExtraOpt);
 }
@@ -1170,21 +1170,16 @@ static std::string GetPluginServerForSDK(llvm::StringRef sdk_path) {
 /// invocation with the concatenated search paths from the blobs.
 /// \returns true if an error was encountered.
 static bool DeserializeAllCompilerFlags(swift::CompilerInvocation &invocation,
-                                        Module &module,
+                                        llvm::StringRef module_name,
+                                        llvm::ArrayRef<StringRef> buffers,
+                                        const PathMappingList &path_map,
                                         bool discover_implicit_search_paths,
                                         const std::string &m_description,
                                         llvm::raw_ostream &error,
                                         bool &got_serialized_options,
                                         bool &found_swift_modules) {
   bool found_validation_errors = false;
   got_serialized_options = false;
-  auto ast_file_datas = module.GetASTData(eLanguageTypeSwift);
-  LOG_PRINTF(GetLog(LLDBLog::Types), "Found %d AST file data entries.",
-             (int)ast_file_datas.size());
-
-  // If no N_AST symbols exist, this is not an error.
-  if (ast_file_datas.empty())
-    return false;
 
   auto &search_path_options = invocation.getSearchPathOptions();
   auto get_override_server = [&](llvm::StringRef plugin_path) -> std::string {
@@ -1257,9 +1252,7 @@ static bool DeserializeAllCompilerFlags(swift::CompilerInvocation &invocation,
 
   // An AST section consists of one or more AST modules, optionally
   // with headers. Iterate over all AST modules.
-  for (auto ast_file_data_sp : ast_file_datas) {
-    llvm::StringRef buf((const char *)ast_file_data_sp->GetBytes(),
-                        ast_file_data_sp->GetByteSize());
+  for (auto buf : buffers) {
     swift::serialization::ValidationInfo info;
     for (; !buf.empty(); buf = buf.substr(info.bytes)) {
       llvm::SmallVector<swift::serialization::SearchPath> searchPaths;
@@ -1274,8 +1267,8 @@ static bool DeserializeAllCompilerFlags(swift::CompilerInvocation &invocation,
       if (invalid_ast || invalid_size || invalid_name) {
         // Validation errors are diagnosed, but not fatal for the context.
         found_validation_errors = true;
-        printASTValidationError(error, info, extended_validation_info, module,
-                                buf, invalid_name, invalid_size);
+        printASTValidationError(error, info, extended_validation_info,
+                                module_name, buf, invalid_name, invalid_size);
         // If there's a size error, quit the loop early, otherwise try the next.
         if (invalid_size)
           break;
@@ -1287,8 +1280,7 @@ static bool DeserializeAllCompilerFlags(swift::CompilerInvocation &invocation,
 
       auto remap = [&](const std::string &path) {
         ConstString remapped;
-        if (module.GetSourceMappingList().RemapPath(ConstString(path),
-                                                    remapped))
+        if (path_map.RemapPath(ConstString(path), remapped))
           return remapped.GetStringRef().str();
         return path;
       };
@@ -1761,17 +1753,37 @@ static std::string GetSDKPathFromDebugInfo(std::string m_description,
   return GetSDKPath(m_description, sdk);
 }
 
+static std::vector<llvm::StringRef>
+GetASTBuffersFromModule(const std::string &m_description,
+                        llvm::ArrayRef<lldb::DataBufferSP> ast_file_datas,
+                        std::string &module_name) {
+  LOG_PRINTF(GetLog(LLDBLog::Types), "Found %d AST file data entries in %s.",
+             (int)ast_file_datas.size(), module_name.c_str());
+  std::vector<llvm::StringRef> buffers;
+  for (auto &data : ast_file_datas)
+    if (data)
+      buffers.push_back(
+          StringRef((const char *)data->GetBytes(), data->GetByteSize()));
+  return buffers;
+}
+
 /// Detect whether a Swift module was "imported" by DWARFImporter.
 /// All this *really* means is that it couldn't be loaded through any
 /// other mechanism.
 static bool IsDWARFImported(swift::ModuleDecl &module) {
-  return std::any_of(module.getFiles().begin(), module.getFiles().end(),
-                     [](swift::FileUnit *file_unit) {
-                       return (file_unit->getKind() ==
-                               swift::FileUnitKind::DWARFModule);
-                     });
+  return llvm::any_of(module.getFiles(), [](swift::FileUnit *file_unit) {
+    return (file_unit->getKind() == swift::FileUnitKind::DWARFModule);
+  });
+}
+
+/// Detect whether this is a proper Swift module.
+static bool IsSerializedAST(swift::ModuleDecl &module) {
+  return llvm::any_of(module.getFiles(), [](swift::FileUnit *file_unit) {
+    return (file_unit->getKind() == swift::FileUnitKind::SerializedAST);
+  });
 }
 
+
 lldb::TypeSystemSP
 SwiftASTContext::CreateInstance(lldb::LanguageType language, Module &module,
                                 TypeSystemSwiftTypeRef &typeref_typesystem,
@@ -1878,15 +1890,24 @@ SwiftASTContext::CreateInstance(lldb::LanguageType language, Module &module,
     bool got_serialized_options = false;
     llvm::SmallString<0> error;
     llvm::raw_svector_ostream errs(error);
-    // Implicit search paths will be discoverd by ValidateSecionModules().
+    // Implicit search paths will be discovered by ValidateSecionModules().
     bool discover_implicit_search_paths = false;
-    if (DeserializeAllCompilerFlags(swift_ast_sp->GetCompilerInvocation(),
-                                    module, discover_implicit_search_paths,
-                                    m_description, errs, got_serialized_options,
-                                    found_swift_modules)) {
-      // Validation errors are not fatal for the context.
-      swift_ast_sp->AddDiagnostic(eDiagnosticSeverityError, errs.str());
-    }
+    auto ast_file_datas = module.GetASTData(eLanguageTypeSwift);
+    std::string module_name = module.GetSpecificationDescription();
+    std::vector<llvm::StringRef> buffers =
+        GetASTBuffersFromModule(m_description, ast_file_datas, module_name);
+
+    // If no N_AST symbols exist, this is not an error.
+    if (!buffers.empty())
+      if (DeserializeAllCompilerFlags(
+              swift_ast_sp->GetCompilerInvocation(),
+              module_name, buffers,
+              module.GetSourceMappingList(), discover_implicit_search_paths,
+              m_description, errs, got_serialized_options,
+              found_swift_modules)) {
+        // Validation errors are not fatal for the context.
+        swift_ast_sp->AddDiagnostic(eDiagnosticSeverityError, errs.str());
+      }
 
     llvm::StringRef serialized_triple =
         swift_ast_sp->GetCompilerInvocation().getTargetTriple();
@@ -2180,13 +2201,21 @@ static void ProcessModule(
   llvm::SmallString<0> error;
   llvm::raw_svector_ostream errs(error);
   swift::CompilerInvocation invocation;
-  if (DeserializeAllCompilerFlags(
-          invocation, *module_sp, discover_implicit_search_paths, m_description,
-          errs, got_serialized_options, found_swift_modules)) {
-    // TODO: After removing DeserializeAllCompilerFlags from
-    //       CreateInstance(per-Module), errs will need to be
-    //       collected here and surfaced.
-  }
+  auto ast_file_datas = module_sp->GetASTData(eLanguageTypeSwift);
+  std::string module_name = module_sp->GetSpecificationDescription();
+  std::vector<llvm::StringRef> buffers =
+      GetASTBuffersFromModule(m_description, ast_file_datas, module_name);
+
+  // If no N_AST symbols exist, this is not an error.
+  if (!buffers.empty())
+    if (DeserializeAllCompilerFlags(
+            invocation, module_name, buffers, module_sp->GetSourceMappingList(),
+            discover_implicit_search_paths, m_description, errs,
+            got_serialized_options, found_swift_modules)) {
+      // TODO: After removing DeserializeAllCompilerFlags from
+      //       CreateInstance(per-Module), errs will need to be
+      //       collected here and surfaced.
+    }
 
   // Copy the interesting deserialized flags to the out parameters.
   const auto &opts = invocation.getSearchPathOptions();
@@ -8316,10 +8345,42 @@ bool SwiftASTContextForExpressions::CacheUserImports(
         LOG_PRINTF(GetLog(LLDBLog::Types | LLDBLog::Expressions),
                    "Performing auto import on found module: %s.\n",
                    module_name.c_str());
-        if (!LoadOneModule(module_info, *this, process_sp,
-                           /*import_dylibs=*/true, error))
+        auto *module_decl = LoadOneModule(module_info, *this, process_sp,
+                                          /*import_dylibs=*/true, error);
+        if (!module_decl)
           return false;
-
+        if (IsSerializedAST(*module_decl)) {
+          // Parse additional search paths from the module.
+          StringRef ast_file = module_decl->getModuleLoadedFilename();
+          if (llvm::sys::path::is_absolute(ast_file)) {
+            auto file_or_err =
+                llvm::MemoryBuffer::getFile(ast_file, /*IsText=*/false,
+                                            /*RequiresNullTerminator=*/false);
+            if (!file_or_err.getError() && file_or_err->get()) {
+              PathMappingList path_remap;
+              llvm::SmallString<0> error;
+              bool found_swift_modules = false;
+              bool got_serialized_options = false;
+              llvm::raw_svector_ostream errs(error);
+              bool discover_implicit_search_paths = false;
+              swift::CompilerInvocation &invocation = GetCompilerInvocation();
+
+              LOG_PRINTF(GetLog(LLDBLog::Types),
+                         "Scanning for search paths in %s",
+                         ast_file.str().c_str());
+              if (DeserializeAllCompilerFlags(
+                      invocation, ast_file, {file_or_err->get()->getBuffer()},
+                      path_remap, discover_implicit_search_paths,
+                      m_description.str().str(), errs, got_serialized_options,
+                      found_swift_modules)) {
+                LOG_PRINTF(GetLog(LLDBLog::Types), "Could not parse %s: %s",
+                           ast_file.str().c_str(), error.str().str().c_str());
+              }
+              if (got_serialized_options)
+                LogConfiguration();
+            }
+          }
+        }
         // How do we tell we are in REPL or playground mode?
         AddHandLoadedModule(module_const_str, attributed_import);
       }

lldb/test/API/lang/swift/deployment_target/TestSwiftDeploymentTarget.py

@@ -66,22 +66,6 @@ def test_swift_deployment_target_from_macho(self):
             self, "break here", lldb.SBFileSpec("main.swift")
         )
         self.expect("expression f", substrs=["i = 23"])
-
-        found_no_ast = False
-        found_triple = False
-        import io
-
-        logfile = io.open(log, "r", encoding="utf-8")
-        for line in logfile:
-            if (
-                'SwiftASTContextForModule("a.out")::DeserializeAllCompilerFlags() -- Found 0 AST file data entries.'
-                in line
-            ):
-                found_no_ast = True
-            if (
-                'SwiftASTContextForModule("a.out")::SetTriple(' in line
-                and "apple-macosx11.0" in line
-            ):
-                found_triple = True
-        self.assertTrue(found_no_ast)
-        self.assertTrue(found_triple)
+        self.filecheck('platform shell cat ""%s"' % log, __file__)
+#       CHECK: SwiftASTContextForExpressions::SetTriple({{.*}}apple-macosx11.0.0
+#       CHECK:  SwiftASTContextForExpressions::RegisterSectionModules("a.out") retrieved 0 AST Data blobs

lldb/test/API/lang/swift/macro/Macro.swift

@@ -0,0 +1 @@
+@freestanding(expression) public macro stringify<T>(_ value: T) -> (T, String) = #externalMacro(module: "MacroImpl", type: "StringifyMacro")

lldb/test/API/lang/swift/macro/Makefile

@@ -1,11 +1,29 @@
-SWIFT_SOURCES := main.swift
-SWIFTFLAGS_EXTRAS = -enable-experimental-feature Macros \
-	            -load-plugin-library $(BUILDDIR)/libMacroImpl.dylib
+SWIFTFLAGS_EXTRAS = -I.
 
-all: libMacroImpl.dylib $(EXE)
+ifneq "$(SWIFT_SOURCES)" "empty.swift"
+
+SWIFTFLAGS_EXTRAS += -load-plugin-library $(BUILDDIR)/libMacroImpl.dylib
+LD_EXTRAS = -L$(BUILDDIR) -lMacro
+
+endif
+
+all: libMacro.dylib libMacroImpl.dylib $(EXE)
 
 include Makefile.rules
 
+libMacro.dylib:
+	$(MAKE) MAKE_DSYM=$(MAKE_DSYM) CC=$(CC) SWIFTC=$(SWIFTC) \
+		ARCH=$(ARCH) DSYMUTIL=$(DSYMUTIL) \
+		VPATH=$(SRCDIR) -I $(SRCDIR) \
+		-f $(THIS_FILE_DIR)/Makefile.rules \
+		DYLIB_SWIFT_SOURCES=Macro.swift \
+		DYLIB_NAME=Macro \
+		DYLIB_ONLY=YES \
+		SWIFT_SOURCES= \
+		SWIFTFLAGS_EXTRAS="-load-plugin-library $(BUILDDIR)/libMacroImpl.dylib" \
+		all
+	$(RM) $(BUILDDIR)/Macro.swiftinterface
+
 libMacroImpl.dylib:
 	$(MAKE) MAKE_DSYM=$(MAKE_DSYM) CC=$(CC) SWIFTC=$(SWIFTC) \
 		ARCH=$(ARCH) DSYMUTIL=$(DSYMUTIL) \

lldb/test/API/lang/swift/macro/TestSwiftMacro.py

@@ -8,14 +8,7 @@
 class TestSwiftMacro(lldbtest.TestBase):
 
     NO_DEBUG_INFO_TESTCASE = True
-
-    @swiftTest
-    # At the time of writing swift/test/Macros/macro_expand.swift is also disabled.
-    @expectedFailureAll(oslist=["linux"])
-    def test(self):
-        """Test Swift macros"""
-        self.build()
-
+    def setupPluginServerForTesting(self):
         # Find the path to the just-built swift-plugin-server.
         # FIXME: this is not very robust.
         def replace_last(old, new, string):
@@ -33,6 +26,15 @@ def replace_last(old, new, string):
             'settings set target.experimental.swift-plugin-server-for-path %s=%s'
             % (self.getBuildDir(), swift_plugin_server))
 
+
+    @swiftTest
+    # At the time of writing swift/test/Macros/macro_expand.swift is also disabled.
+    @expectedFailureAll(oslist=["linux"])
+    def testDebugging(self):
+        """Test Swift macros"""
+        self.build(dictionary={'SWIFT_SOURCES': 'main.swift'})
+        self.setupPluginServerForTesting()
+
         target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
             self, "break here", lldb.SBFileSpec("main.swift")
         )
@@ -53,3 +55,24 @@ def replace_last(old, new, string):
         # Make sure we can set a symbolic breakpoint on a macro.
         b = target.BreakpointCreateByName("stringify")
         self.assertGreaterEqual(b.GetNumLocations(), 1)
+
+    @swiftTest
+    # At the time of writing swift/test/Macros/macro_expand.swift is also disabled.
+    @expectedFailureAll(oslist=["linux"])
+    def testInteractive(self):
+        """Test Swift macros that are loaded via a user-initiated import"""
+        self.build(dictionary={'SWIFT_SOURCES': 'empty.swift'})
+        self.setupPluginServerForTesting()
+        target, process, thread, bkpt = lldbutil.run_to_name_breakpoint(
+            self, "main"
+        )
+
+        types_log = self.getBuildArtifact('types.log')
+        self.expect('log enable lldb types -f "%s"' % types_log)
+        self.expect('expression -- import Macro')
+        self.expect('expression -- #stringify(1)', substrs=['0 = 1', '1 = "1"'])
+        self.filecheck('platform shell cat "%s"' % types_log, __file__)
+#       CHECK: CacheUserImports(){{.*}}: Macro.
+#       CHECK: SwiftASTContextForExpressions::LoadOneModule(){{.*}}Imported module Macro from {kind = Serialized Swift AST, filename = "{{.*}}Macro.swiftmodule";}
+#       CHECK: CacheUserImports(){{.*}}Scanning for search paths in{{.*}}Macro.swiftmodule
+#       CHECK: SwiftASTContextForExpressions::LogConfiguration(){{.*}} -external-plugin-path {{.*}}/lang/swift/macro/{{.*}}#{{.*}}/swift-plugin-server

lldb/test/API/lang/swift/macro/empty.swift

@@ -0,0 +1 @@
+print("Hello World")

lldb/test/API/lang/swift/macro/main.swift

@@ -1,4 +1,4 @@
-@freestanding(expression) macro stringify<T>(_ value: T) -> (T, String) = #externalMacro(module: "MacroImpl", type: "StringifyMacro")
+import Macro
 
 func testStringify(a: Int, b: Int) {
   print("break here")