Merge pull request #14814 from jrose-apple/dump-truck

Dump (some) filelists in the PrettyStackTrace

Author: jrose-apple

include/swift/Basic/PrettyStackTrace.h

@@ -16,6 +16,10 @@
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/ADT/StringRef.h"
 
+namespace llvm {
+  class MemoryBuffer;
+}
+
 namespace swift {
 
 /// A PrettyStackTraceEntry for performing an action involving a StringRef.
@@ -31,6 +35,15 @@ class PrettyStackTraceStringAction : public llvm::PrettyStackTraceEntry {
   void print(llvm::raw_ostream &OS) const override;
 };
 
+/// A PrettyStackTraceEntry to dump the contents of a file.
+class PrettyStackTraceFileContents : public llvm::PrettyStackTraceEntry {
+  const llvm::MemoryBuffer &Buffer;
+public:
+  explicit PrettyStackTraceFileContents(const llvm::MemoryBuffer &buffer)
+    : Buffer(buffer) {}
+  void print(llvm::raw_ostream &OS) const override;
+};
+
 } // end namespace swift
 
 #endif // SWIFT_BASIC_PRETTYSTACKTRACE_H

include/swift/Frontend/ArgsToFrontendInputsConverter.h

@@ -42,21 +42,28 @@ namespace swift {
 class ArgsToFrontendInputsConverter {
   DiagnosticEngine &Diags;
   const llvm::opt::ArgList &Args;
-  FrontendInputsAndOutputs &InputsAndOutputs;
 
   llvm::opt::Arg const *const FilelistPathArg;
   llvm::opt::Arg const *const PrimaryFilelistPathArg;
 
-  SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 4> BuffersToKeepAlive;
+  /// A place to keep alive any buffers that are loaded as part of setting up
+  /// the frontend inputs.
+  SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 4> ConfigurationFileBuffers;
 
   llvm::SetVector<StringRef> Files;
 
 public:
   ArgsToFrontendInputsConverter(DiagnosticEngine &diags,
-                                const llvm::opt::ArgList &args,
-                                FrontendInputsAndOutputs &inputsAndOutputs);
+                                const llvm::opt::ArgList &args);
 
-  bool convert();
+  /// Produces a FrontendInputsAndOutputs object with the inputs populated from
+  /// the arguments the converter was initialized with.
+  ///
+  /// \param buffers If present, buffers read in the processing of the frontend
+  /// inputs will be saved here. These should only be used for debugging
+  /// purposes.
+  Optional<FrontendInputsAndOutputs> convert(
+      SmallVectorImpl<std::unique_ptr<llvm::MemoryBuffer>> *buffers);
 
 private:
   bool enforceFilelistExclusion();
@@ -66,11 +73,19 @@ class ArgsToFrontendInputsConverter {
                              llvm::function_ref<void(StringRef)> fn);
   bool addFile(StringRef file);
   Optional<std::set<StringRef>> readPrimaryFiles();
-  std::set<StringRef>
+
+  /// Returns the newly set-up FrontendInputsAndOutputs, as well as a set of
+  /// any unused primary files (those that do not correspond to an input).
+  std::pair<FrontendInputsAndOutputs, std::set<StringRef>>
   createInputFilesConsumingPrimaries(std::set<StringRef> primaryFiles);
-  bool checkForMissingPrimaryFiles(std::set<StringRef> primaryFiles);
 
-  bool isSingleThreadedWMO() const;
+  /// Emits an error for each file in \p unusedPrimaryFiles.
+  ///
+  /// \returns true if \p unusedPrimaryFiles is non-empty.
+  bool diagnoseUnusedPrimaryFiles(std::set<StringRef> unusedPrimaryFiles);
+
+  bool
+  isSingleThreadedWMO(const FrontendInputsAndOutputs &inputsAndOutputs) const;
 };
 
 } // namespace swift

include/swift/Frontend/ArgsToFrontendOptionsConverter.h

@@ -69,7 +69,13 @@ class ArgsToFrontendOptionsConverter {
                                  FrontendOptions &Opts)
       : Diags(Diags), Args(Args), Opts(Opts) {}
 
-  bool convert();
+  /// Populates the FrontendOptions the converter was initialized with.
+  ///
+  /// \param buffers If present, buffers read in the processing of the frontend
+  /// options will be saved here. These should only be used for debugging
+  /// purposes.
+  bool convert(
+      SmallVectorImpl<std::unique_ptr<llvm::MemoryBuffer>> *buffers);
 
   static FrontendOptions::ActionType
   determineRequestedAction(const llvm::opt::ArgList &);

include/swift/Frontend/Frontend.h

@@ -86,11 +86,19 @@ class CompilerInvocation {
   /// default values given the /absence/ of a flag. This is because \c parseArgs
   /// may be used to modify an already partially configured invocation.
   ///
+  /// Any configuration files loaded as a result of parsing arguments will be
+  /// stored in \p ConfigurationFileBuffers, if non-null. The contents of these
+  /// buffers should \e not be interpreted by the caller; they are only present
+  /// in order to make it possible to reproduce how these arguments were parsed
+  /// if the compiler ends up crashing or exhibiting other bad behavior.
+  ///
   /// If non-empty, relative search paths are resolved relative to
   /// \p workingDirectory.
   ///
   /// \returns true if there was an error, false on success.
   bool parseArgs(ArrayRef<const char *> Args, DiagnosticEngine &Diags,
+                 SmallVectorImpl<std::unique_ptr<llvm::MemoryBuffer>>
+                     *ConfigurationFileBuffers = nullptr,
                  StringRef workingDirectory = {});
 
   /// Sets specific options based on the given serialized Swift binary data.

lib/Basic/PrettyStackTrace.cpp

@@ -17,10 +17,19 @@
 
 #include "swift/Basic/PrettyStackTrace.h"
 #include "swift/Basic/QuotedString.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace swift;
 
 void PrettyStackTraceStringAction::print(llvm::raw_ostream &out) const {
   out << "While " << Action << ' ' << QuotedString(TheString) << '\n';
 }
+
+void PrettyStackTraceFileContents::print(llvm::raw_ostream &out) const {
+  out << "Contents of " << Buffer.getBufferIdentifier() << ":\n---\n"
+      << Buffer.getBuffer();
+  if (!Buffer.getBuffer().endswith("\n"))
+    out << '\n';
+  out << "---\n";
+}

lib/Frontend/ArgsToFrontendInputsConverter.cpp

@@ -13,6 +13,7 @@
 #include "swift/Frontend/ArgsToFrontendInputsConverter.h"
 
 #include "swift/AST/DiagnosticsFrontend.h"
+#include "swift/Basic/Defer.h"
 #include "swift/Frontend/ArgsToFrontendOutputsConverter.h"
 #include "swift/Frontend/FrontendOptions.h"
 #include "swift/Option/Options.h"
@@ -30,33 +31,47 @@ using namespace swift;
 using namespace llvm::opt;
 
 ArgsToFrontendInputsConverter::ArgsToFrontendInputsConverter(
-    DiagnosticEngine &diags, const ArgList &args,
-    FrontendInputsAndOutputs &inputsAndOutputs)
-    : Diags(diags), Args(args), InputsAndOutputs(inputsAndOutputs),
+    DiagnosticEngine &diags, const ArgList &args)
+    : Diags(diags), Args(args),
       FilelistPathArg(args.getLastArg(options::OPT_filelist)),
       PrimaryFilelistPathArg(args.getLastArg(options::OPT_primary_filelist)) {}
 
-bool ArgsToFrontendInputsConverter::convert() {
+Optional<FrontendInputsAndOutputs> ArgsToFrontendInputsConverter::convert(
+    SmallVectorImpl<std::unique_ptr<llvm::MemoryBuffer>> *buffers) {
+  SWIFT_DEFER {
+    if (buffers) {
+      std::move(ConfigurationFileBuffers.begin(),
+                ConfigurationFileBuffers.end(),
+                std::back_inserter(*buffers));
+      // Clearing the original list of buffers isn't strictly necessary, but
+      // makes the behavior more sensible if we were to call convert() again.
+      ConfigurationFileBuffers.clear();
+    }
+  };
+
   if (enforceFilelistExclusion())
-    return true;
+    return None;
+
   if (FilelistPathArg ? readInputFilesFromFilelist()
                       : readInputFilesFromCommandLine())
-    return true;
+    return None;
   Optional<std::set<StringRef>> primaryFiles = readPrimaryFiles();
   if (!primaryFiles)
-    return true;
-  std::set<StringRef> unusedPrimaryFiles =
+    return None;
+
+  FrontendInputsAndOutputs result;
+  std::set<StringRef> unusedPrimaryFiles;
+  std::tie(result, unusedPrimaryFiles) =
       createInputFilesConsumingPrimaries(*primaryFiles);
 
-  if (checkForMissingPrimaryFiles(unusedPrimaryFiles))
-    return true;
+  if (diagnoseUnusedPrimaryFiles(unusedPrimaryFiles))
+    return None;
 
   // Must be set before iterating over inputs needing outputs.
-  InputsAndOutputs.setIsSingleThreadedWMO(isSingleThreadedWMO());
-
-  InputsAndOutputs.setBypassBatchModeChecks(
+  result.setBypassBatchModeChecks(
       Args.hasArg(options::OPT_bypass_batch_mode_checks));
-  return false;
+
+  return std::move(result);
 }
 
 bool ArgsToFrontendInputsConverter::enforceFilelistExclusion() {
@@ -114,7 +129,7 @@ bool ArgsToFrontendInputsConverter::forAllFilesInFilelist(
        llvm::make_range(llvm::line_iterator(*filelistBufferOrError->get()),
                         llvm::line_iterator()))
     fn(file);
-  BuffersToKeepAlive.push_back(std::move(*filelistBufferOrError));
+  ConfigurationFileBuffers.push_back(std::move(*filelistBufferOrError));
   return false;
 }
 
@@ -137,34 +152,38 @@ ArgsToFrontendInputsConverter::readPrimaryFiles() {
   return primaryFiles;
 }
 
-std::set<StringRef>
+std::pair<FrontendInputsAndOutputs, std::set<StringRef>>
 ArgsToFrontendInputsConverter::createInputFilesConsumingPrimaries(
     std::set<StringRef> primaryFiles) {
+  bool hasAnyPrimaryFiles = !primaryFiles.empty();
+
+  FrontendInputsAndOutputs result;
   for (auto &file : Files) {
     bool isPrimary = primaryFiles.count(file) > 0;
-    InputsAndOutputs.addInput(InputFile(file, isPrimary));
+    result.addInput(InputFile(file, isPrimary));
     if (isPrimary)
       primaryFiles.erase(file);
   }
-  return primaryFiles;
+
+  if (!Files.empty() && !hasAnyPrimaryFiles) {
+    Optional<std::vector<std::string>> userSuppliedNamesOrErr =
+        OutputFilesComputer::getOutputFilenamesFromCommandLineOrFilelist(Args,
+                                                                         Diags);
+    if (userSuppliedNamesOrErr && userSuppliedNamesOrErr->size() == 1)
+      result.setIsSingleThreadedWMO(true);
+  }
+
+  return {std::move(result), std::move(primaryFiles)};
 }
 
-bool ArgsToFrontendInputsConverter::checkForMissingPrimaryFiles(
+bool ArgsToFrontendInputsConverter::diagnoseUnusedPrimaryFiles(
     std::set<StringRef> primaryFiles) {
   for (auto &file : primaryFiles) {
     // Catch "swiftc -frontend -c -filelist foo -primary-file
     // some-file-not-in-foo".
-    assert(FilelistPathArg && "Missing primary with no filelist");
+    assert(FilelistPathArg && "Unused primary with no filelist");
     Diags.diagnose(SourceLoc(), diag::error_primary_file_not_found, file,
                    FilelistPathArg->getValue());
   }
   return !primaryFiles.empty();
 }
-
-bool ArgsToFrontendInputsConverter::isSingleThreadedWMO() const {
-  Optional<std::vector<std::string>> userSuppliedNamesOrErr =
-      OutputFilesComputer::getOutputFilenamesFromCommandLineOrFilelist(Args,
-                                                                       Diags);
-  return InputsAndOutputs.hasInputs() && !InputsAndOutputs.hasPrimaryInputs() &&
-         userSuppliedNamesOrErr && userSuppliedNamesOrErr->size() == 1;
-}

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -45,7 +45,8 @@ static void debugFailWithAssertion() {
 LLVM_ATTRIBUTE_NOINLINE
 static void debugFailWithCrash() { LLVM_BUILTIN_TRAP; }
 
-bool ArgsToFrontendOptionsConverter::convert() {
+bool ArgsToFrontendOptionsConverter::convert(
+    SmallVectorImpl<std::unique_ptr<llvm::MemoryBuffer>> *buffers) {
   using namespace options;
 
   handleDebugCrashGroupArguments();
@@ -93,9 +94,11 @@ bool ArgsToFrontendOptionsConverter::convert() {
 
   computeDumpScopeMapLocations();
 
-  if (ArgsToFrontendInputsConverter(Diags, Args, Opts.InputsAndOutputs)
-          .convert())
+  Optional<FrontendInputsAndOutputs> inputsAndOutputs =
+      ArgsToFrontendInputsConverter(Diags, Args).convert(buffers);
+  if (!inputsAndOutputs)
     return true;
+  Opts.InputsAndOutputs = std::move(inputsAndOutputs).getValue();
 
   Opts.RequestedAction = determineRequestedAction(Args);
 

lib/Frontend/CompilerInvocation.cpp

@@ -81,9 +81,11 @@ SourceFileKind CompilerInvocation::getSourceFileKind() const {
   llvm_unreachable("Unhandled InputFileKind in switch.");
 }
 
-static bool ParseFrontendArgs(FrontendOptions &opts, ArgList &args,
-                              DiagnosticEngine &diags) {
-  return ArgsToFrontendOptionsConverter(diags, args, opts).convert();
+static bool ParseFrontendArgs(
+    FrontendOptions &opts, ArgList &args, DiagnosticEngine &diags,
+    SmallVectorImpl<std::unique_ptr<llvm::MemoryBuffer>> *buffers) {
+  ArgsToFrontendOptionsConverter converter(diags, args, opts);
+  return converter.convert(buffers);
 }
 
 static void diagnoseSwiftVersion(Optional<version::Version> &vers, Arg *verArg,
@@ -978,9 +980,12 @@ bool ParseMigratorArgs(MigratorOptions &Opts,
   return false;
 }
 
-bool CompilerInvocation::parseArgs(ArrayRef<const char *> Args,
-                                   DiagnosticEngine &Diags,
-                                   StringRef workingDirectory) {
+bool CompilerInvocation::parseArgs(
+    ArrayRef<const char *> Args,
+    DiagnosticEngine &Diags,
+    SmallVectorImpl<std::unique_ptr<llvm::MemoryBuffer>>
+        *ConfigurationFileBuffers,
+    StringRef workingDirectory) {
   using namespace options;
 
   if (Args.empty())
@@ -1006,7 +1011,8 @@ bool CompilerInvocation::parseArgs(ArrayRef<const char *> Args,
     return true;
   }
 
-  if (ParseFrontendArgs(FrontendOpts, ParsedArgs, Diags)) {
+  if (ParseFrontendArgs(FrontendOpts, ParsedArgs, Diags,
+                        ConfigurationFileBuffers)) {
     return true;
   }
 

lib/FrontendTool/FrontendTool.cpp

@@ -41,6 +41,7 @@
 #include "swift/Basic/JSONSerialization.h"
 #include "swift/Basic/LLVMContext.h"
 #include "swift/Basic/LLVMInitialize.h"
+#include "swift/Basic/PrettyStackTrace.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/Basic/Statistic.h"
 #include "swift/Basic/Timer.h"
@@ -1661,10 +1662,32 @@ int swift::performFrontend(ArrayRef<const char *> Args,
   llvm::sys::fs::current_path(workingDirectory);
 
   // Parse arguments.
-  if (Invocation.parseArgs(Args, Instance->getDiags(), workingDirectory)) {
+  SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 4> configurationFileBuffers;
+  if (Invocation.parseArgs(Args, Instance->getDiags(),
+                           &configurationFileBuffers, workingDirectory)) {
     return finishDiagProcessing(1);
   }
 
+  // Make an array of PrettyStackTrace objects to dump the configuration files
+  // we used to parse the arguments. These are RAII objects, so they and the
+  // buffers they refer to must be kept alive in order to be useful. (That is,
+  // we want them to be alive for the entire rest of performFrontend.)
+  //
+  // This can't be a SmallVector or similar because PrettyStackTraces can't be
+  // moved (or copied)...and it can't be an array of non-optionals because
+  // PrettyStackTraces can't be default-constructed. So we end up with a
+  // dynamically-sized array of optional PrettyStackTraces, which get
+  // initialized by iterating over the buffers we collected above.
+  auto configurationFileStackTraces =
+      llvm::make_unique<Optional<PrettyStackTraceFileContents>[]>(
+        configurationFileBuffers.size());
+  for_each(configurationFileBuffers.begin(), configurationFileBuffers.end(),
+           &configurationFileStackTraces[0],
+           [](const std::unique_ptr<llvm::MemoryBuffer> &buffer,
+              Optional<PrettyStackTraceFileContents> &trace) {
+    trace.emplace(*buffer);
+  });
+
   // Setting DWARF Version depend on platform
   IRGenOptions &IRGenOpts = Invocation.getIRGenOptions();
   IRGenOpts.DWARFVersion = swift::DWARFVersion;

test/Frontend/crash.swift

@@ -0,0 +1,14 @@
+// RUN: echo %s > %t.filelist.txt
+// RUN: not --crash %target-swift-frontend -typecheck -debug-crash-after-parse -filelist %t.filelist.txt 2>&1 | %FileCheck %s
+
+// Check that we see the contents of the input file list in the crash log.
+// CHECK-LABEL: Stack dump
+// CHECK-NEXT: Program arguments: {{.*swift(c?)}} -frontend
+// CHECK-NEXT: Contents of {{.*}}.filelist.txt:
+// CHECK-NEXT: ---
+// CHECK-NEXT: test/Frontend/crash.swift{{$}}
+// CHECK-NEXT: ---
+
+func anchor() {}
+anchor()
+