Merge pull request #33569 from nkcsgexi/batch-scanning-protocol

DependenciesScanner: implement protocol for batch module dependencies scan

Author: nkcsgexi

include/swift/AST/ASTContext.h

@@ -739,6 +739,12 @@ class ASTContext final {
       ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate);
 
+  /// Retrieve the module dependencies for the Swift module with the given name.
+  Optional<ModuleDependencies> getSwiftModuleDependencies(
+      StringRef moduleName,
+      ModuleDependenciesCache &cache,
+      InterfaceSubContextDelegate &delegate);
+
   /// Load extensions to the given nominal type from the external
   /// module loaders.
   ///

include/swift/AST/DiagnosticsFrontend.def

@@ -274,6 +274,14 @@ ERROR(placeholder_dependency_module_map_corrupted,none,
       "Swift placeholder dependency module map from %0 is malformed",
       (StringRef))
 
+ERROR(batch_scan_input_file_missing,none,
+      "cannot open batch dependencies scan input file from %0",
+      (StringRef))
+
+ERROR(batch_scan_input_file_corrupted,none,
+      "batch dependencies scan input file from %0 is malformed",
+      (StringRef))
+
 REMARK(default_previous_install_name, none,
       "default previous install name for %0 is %1", (StringRef, StringRef))
 

include/swift/AST/SearchPathOptions.h

@@ -93,6 +93,9 @@ class SearchPathOptions {
 
   /// A map of placeholder Swift module dependency information.
   std::string PlaceholderDependencyModuleMap;
+
+  /// A file containing modules we should perform batch scanning.
+  std::string BatchScanInputFilePath;
 private:
   static StringRef
   pathStringFromFrameworkSearchPath(const FrameworkSearchPath &next) {

include/swift/Option/FrontendOptions.td

@@ -228,6 +228,10 @@ def explict_swift_module_map
 def placeholder_dependency_module_map
   : Separate<["-"], "placeholder-dependency-module-map-file">, MetaVarName<"<path>">,
     HelpText<"Specify a JSON file containing information of external Swift module dependencies">;
+
+def batch_scan_input_file
+  : Separate<["-"], "batch-scan-input-file">, MetaVarName<"<path>">,
+    HelpText<"Specify a JSON file containing modules to perform batch dependencies scanning">;
 }
 
 

lib/AST/ASTContext.cpp

@@ -1516,6 +1516,21 @@ Optional<ModuleDependencies> ASTContext::getModuleDependencies(
   return None;
 }
 
+Optional<ModuleDependencies>
+ASTContext::getSwiftModuleDependencies(StringRef moduleName,
+                                       ModuleDependenciesCache &cache,
+                                       InterfaceSubContextDelegate &delegate) {
+  for (auto &loader : getImpl().ModuleLoaders) {
+    if (loader.get() == getImpl().TheClangModuleLoader)
+      continue;
+
+    if (auto dependencies = loader->getModuleDependencies(moduleName, cache,
+                                                          delegate))
+      return dependencies;
+  }
+  return None;
+}
+
 void ASTContext::loadExtensions(NominalTypeDecl *nominal,
                                 unsigned previousGeneration) {
   PrettyStackTraceDecl stackTrace("loading extensions for", nominal);

lib/Frontend/CompilerInvocation.cpp

@@ -926,7 +926,8 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts,
   }
   if (const Arg *A = Args.getLastArg(OPT_placeholder_dependency_module_map))
     Opts.PlaceholderDependencyModuleMap = A->getValue();
-
+  if (const Arg *A = Args.getLastArg(OPT_batch_scan_input_file))
+    Opts.BatchScanInputFilePath = A->getValue();
   // Opts.RuntimeIncludePath is set by calls to
   // setRuntimeIncludePath() or setMainExecutablePath().
   // Opts.RuntimeImportPath is set by calls to

lib/FrontendTool/FrontendTool.cpp

@@ -1834,8 +1834,14 @@ static bool performCompile(CompilerInstance &Instance,
     return finishPipeline(Context.hadError());
   }
 
-  if (Action == FrontendOptions::ActionType::ScanDependencies)
-    return finishPipeline(scanDependencies(Instance));
+  if (Action == FrontendOptions::ActionType::ScanDependencies) {
+    auto batchScanInput = Instance.getASTContext().SearchPathOpts.BatchScanInputFilePath;
+    if (batchScanInput.empty())
+      return finishPipeline(scanDependencies(Instance));
+    else
+      return finishPipeline(batchScanModuleDependencies(Instance,
+                                                        batchScanInput));
+  }
 
   if (Action == FrontendOptions::ActionType::ScanClangDependencies)
     return finishPipeline(scanClangDependencies(Instance));

lib/FrontendTool/ScanDependencies.cpp

@@ -32,12 +32,103 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/StringSaver.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/YAMLParser.h"
 #include <set>
 
 using namespace swift;
+using namespace llvm::yaml;
 
 namespace {
+struct BatchScanInput {
+  StringRef moduleName;
+  StringRef arguments;
+  StringRef outputPath;
+  bool isSwift;
+};
+
+static std::string getScalaNodeText(Node *N) {
+  SmallString<32> Buffer;
+  return cast<ScalarNode>(N)->getValue(Buffer).str();
+}
+
+/// Parse an entry like this, where the "platforms" key-value pair is optional:
+///  {
+///     "swiftModuleName": "Foo",
+///     "arguments": "-target 10.15",
+///     "output": "../Foo.json"
+///  },
+static bool parseBatchInputEntries(ASTContext &Ctx, llvm::StringSaver &saver,
+                                   Node *Node, std::vector<BatchScanInput> &result) {
+  auto *SN = cast<SequenceNode>(Node);
+  if (!SN)
+    return true;
+  for (auto It = SN->begin(); It != SN->end(); ++It) {
+    auto *MN = cast<MappingNode>(&*It);
+    BatchScanInput entry;
+    Optional<std::set<int8_t>> Platforms;
+    for (auto &Pair: *MN) {
+      auto Key = getScalaNodeText(Pair.getKey());
+      auto* Value = Pair.getValue();
+      if (Key == "clangModuleName") {
+        entry.moduleName = saver.save(getScalaNodeText(Value));
+        entry.isSwift = false;
+      } else if (Key == "swiftModuleName") {
+        entry.moduleName = saver.save(getScalaNodeText(Value));
+        entry.isSwift = true;
+      } else if (Key == "arguments") {
+        entry.arguments = saver.save(getScalaNodeText(Value));
+      } else if (Key == "output") {
+        entry.outputPath = saver.save(getScalaNodeText(Value));
+      } else {
+        // Future proof.
+        continue;
+      }
+    }
+    if (entry.moduleName.empty())
+      return true;
+    if (entry.outputPath.empty())
+      return true;
+    result.emplace_back(std::move(entry));
+  }
+  return false;
+}
 
+static Optional<std::vector<BatchScanInput>>
+parseBatchScanInputFile(ASTContext &ctx, StringRef batchInputPath,
+                        llvm::StringSaver &saver) {
+  assert(!batchInputPath.empty());
+  namespace yaml = llvm::yaml;
+  std::vector<BatchScanInput> result;
+
+  // Load the input file.
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileBufOrErr =
+    llvm::MemoryBuffer::getFile(batchInputPath);
+  if (!FileBufOrErr) {
+    ctx.Diags.diagnose(SourceLoc(), diag::batch_scan_input_file_missing,
+                       batchInputPath);
+    return None;
+  }
+  StringRef Buffer = FileBufOrErr->get()->getBuffer();
+
+  // Use a new source manager instead of the one from ASTContext because we
+  // don't want the Json file to be persistent.
+  SourceManager SM;
+  yaml::Stream Stream(llvm::MemoryBufferRef(Buffer, batchInputPath),
+                      SM.getLLVMSourceMgr());
+  for (auto DI = Stream.begin(); DI != Stream.end(); ++ DI) {
+    assert(DI != Stream.end() && "Failed to read a document");
+    yaml::Node *N = DI->getRoot();
+    assert(N && "Failed to find a root");
+    if (parseBatchInputEntries(ctx, saver, N, result)) {
+      ctx.Diags.diagnose(SourceLoc(), diag::batch_scan_input_file_corrupted,
+                         batchInputPath);
+      return None;
+    }
+  }
+  return result;
+}
 }
 
 /// Find all of the imported Clang modules starting with the given module name.
@@ -533,15 +624,17 @@ static bool diagnoseCycle(CompilerInstance &instance,
   return false;
 }
 
-bool swift::scanClangDependencies(CompilerInstance &instance) {
+static bool scanModuleDependencies(CompilerInstance &instance,
+                                   StringRef moduleName,
+                                   StringRef arguments,
+                                   bool isClang,
+                                   StringRef outputPath) {
   ASTContext &ctx = instance.getASTContext();
-  ModuleDecl *mainModule = instance.getMainModule();
   auto &FEOpts = instance.getInvocation().getFrontendOptions();
   ModuleInterfaceLoaderOptions LoaderOpts(FEOpts);
   auto ModuleCachePath = getModuleCachePathFromClang(ctx
     .getClangModuleLoader()->getClangInstance());
 
-  StringRef mainModuleName = mainModule->getNameStr();
   llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
                   std::set<ModuleDependencyID>> allModules;
   // Create the module dependency cache.
@@ -555,16 +648,23 @@ bool swift::scanClangDependencies(CompilerInstance &instance) {
                                               FEOpts.PrebuiltModuleCachePath,
                                               FEOpts.SerializeModuleInterfaceDependencyHashes,
                                               FEOpts.shouldTrackSystemDependencies());
-  // Loading the clang module using Clang importer.
-  // This action will populate the cache with the main module's dependencies.
-  auto rootDeps = static_cast<ClangImporter*>(ctx.getClangModuleLoader())
-    ->getModuleDependencies(mainModuleName, cache, ASTDelegate);
+  Optional<ModuleDependencies> rootDeps;
+  if (isClang) {
+    // Loading the clang module using Clang importer.
+    // This action will populate the cache with the main module's dependencies.
+    rootDeps = ctx.getModuleDependencies(moduleName, /*IsClang*/true, cache,
+                                         ASTDelegate);
+  } else {
+    // Loading the swift module's dependencies.
+    rootDeps = ctx.getSwiftModuleDependencies(moduleName, cache, ASTDelegate);
+  }
   if (!rootDeps.hasValue()) {
     // We cannot find the clang module, abort.
     return true;
   }
   // Add the main module.
-  allModules.insert({mainModuleName.str(), ModuleDependenciesKind::Clang});
+  allModules.insert({moduleName.str(), isClang ? ModuleDependenciesKind::Clang:
+    ModuleDependenciesKind::Swift});
 
   // Explore the dependencies of every module.
   for (unsigned currentModuleIdx = 0;
@@ -576,13 +676,38 @@ bool swift::scanClangDependencies(CompilerInstance &instance) {
     allModules.insert(discoveredModules.begin(), discoveredModules.end());
   }
   // Write out the JSON description.
-  std::string path = FEOpts.InputsAndOutputs.getSingleOutputFilename();
   std::error_code EC;
-  llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None);
+  llvm::raw_fd_ostream out(outputPath, EC, llvm::sys::fs::F_None);
   writeJSON(out, instance, cache, ASTDelegate, allModules.getArrayRef());
   return false;
 }
 
+bool swift::scanClangDependencies(CompilerInstance &instance) {
+  return scanModuleDependencies(instance,
+                                instance.getMainModule()->getNameStr(),
+                                StringRef(),
+                                /*isClang*/true,
+                                instance.getInvocation().getFrontendOptions()
+                                  .InputsAndOutputs.getSingleOutputFilename());
+}
+
+bool swift::batchScanModuleDependencies(CompilerInstance &instance,
+                                      llvm::StringRef batchInputFile) {
+  (void)instance.getMainModule();
+  llvm::BumpPtrAllocator alloc;
+  llvm::StringSaver saver(alloc);
+  auto results = parseBatchScanInputFile(instance.getASTContext(),
+                                         batchInputFile, saver);
+  if (!results.hasValue())
+    return true;
+  for (auto &entry: *results) {
+    if (scanModuleDependencies(instance, entry.moduleName, entry.arguments,
+                               !entry.isSwift, entry.outputPath))
+      return true;
+  }
+  return false;
+}
+
 bool swift::scanDependencies(CompilerInstance &instance) {
   ASTContext &Context = instance.getASTContext();
   ModuleDecl *mainModule = instance.getMainModule();

lib/FrontendTool/ScanDependencies.h

@@ -13,10 +13,15 @@
 #ifndef SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H
 #define SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H
 
+#include "llvm/ADT/StringRef.h"
+
 namespace swift {
 
 class CompilerInstance;
 
+/// Batch scan the dependencies for modules specified in \c batchInputFile.
+bool batchScanModuleDependencies(CompilerInstance &instance, llvm::StringRef batchInputFile);
+
 /// Scans the dependencies of the main module of \c instance.
 bool scanDependencies(CompilerInstance &instance);
 

test/ScanDependencies/batch_module_scan.swift

@@ -0,0 +1,39 @@
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/inputs)
+// RUN: %empty-directory(%t/outputs)
+// RUN: mkdir -p %t/clang-module-cache
+
+// RUN: echo "[{" > %/t/inputs/input.json
+// RUN: echo "\"swiftModuleName\": \"F\"," >> %/t/inputs/input.json
+// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json
+// RUN: echo "\"output\": \"%/t/outputs/F.swiftmodule.json\"" >> %/t/inputs/input.json
+// RUN: echo "}," >> %/t/inputs/input.json
+// RUN: echo "{" >> %/t/inputs/input.json
+// RUN: echo "\"clangModuleName\": \"F\"," >> %/t/inputs/input.json
+// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json
+// RUN: echo "\"output\": \"%/t/outputs/F.pcm.json\"" >> %/t/inputs/input.json
+// RUN: echo "}]" >> %/t/inputs/input.json
+
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 -batch-scan-input-file %/t/inputs/input.json
+
+// Check the contents of the JSON output
+// RUN: %FileCheck %s -check-prefix=CHECK-PCM < %t/outputs/F.pcm.json
+// RUN: %FileCheck %s -check-prefix=CHECK-SWIFT < %t/outputs/F.swiftmodule.json
+
+// CHECK-PCM: 		{
+// CHECK-PCM-NEXT:  "mainModuleName": "F",
+// CHECK-PCM-NEXT:  "modules": [
+// CHECK-PCM-NEXT:    {
+// CHECK-PCM-NEXT:      "clang": "F"
+// CHECK-PCM-NEXT:    },
+// CHECK-PCM-NEXT:    {
+// CHECK-PCM-NEXT:      "modulePath": "F.pcm",
+
+// CHECK-SWIFT: {
+// CHECK-SWIFT-NEXT:  "mainModuleName": "F",
+// CHECK-SWIFT-NEXT:  "modules": [
+// CHECK-SWIFT-NEXT:    {
+// CHECK-SWIFT-NEXT:      "swift": "F"
+// CHECK-SWIFT-NEXT:    },
+// CHECK-SWIFT-NEXT:    {
+// CHECK-SWIFT-NEXT:      "modulePath": "F.swiftmodule",