Merge pull request #16362 from dabelknap/frontend_responsefile

Wrap Command Line Arguments in a Response File if System Limits are Exceeded

https://bugs.swift.org/browse/SR-4517

Author: jrose-apple

include/swift/Driver/Job.h

@@ -271,6 +271,14 @@ class Job {
   /// Whether the job wants a list of input or output files created.
   std::vector<FilelistInfo> FilelistFileInfos;
 
+  /// Response file path
+  const char *ResponseFilePath;
+
+  /// This contains a single argument pointing to the response file path with
+  /// the '@' prefix.
+  /// The argument string must be kept alive as long as the Job is alive.
+  const char *ResponseFileArg;
+
   /// The modification time of the main input file, if any.
   llvm::sys::TimePoint<> InputModTime = llvm::sys::TimePoint<>::max();
 
@@ -281,12 +289,16 @@ class Job {
       const char *Executable,
       llvm::opt::ArgStringList Arguments,
       EnvironmentVector ExtraEnvironment = {},
-      std::vector<FilelistInfo> Infos = {})
+      std::vector<FilelistInfo> Infos = {},
+      const char *ResponseFilePath = nullptr,
+      const char *ResponseFileArg = nullptr)
       : SourceAndCondition(&Source, Condition::Always),
         Inputs(std::move(Inputs)), Output(std::move(Output)),
         Executable(Executable), Arguments(std::move(Arguments)),
         ExtraEnvironment(std::move(ExtraEnvironment)),
-        FilelistFileInfos(std::move(Infos)) {}
+        FilelistFileInfos(std::move(Infos)),
+        ResponseFilePath(ResponseFilePath),
+        ResponseFileArg(ResponseFileArg) {}
 
   virtual ~Job();
 
@@ -296,7 +308,9 @@ class Job {
 
   const char *getExecutable() const { return Executable; }
   const llvm::opt::ArgStringList &getArguments() const { return Arguments; }
+  ArrayRef<const char *> getResponseFileArg() const { return ResponseFileArg; }
   ArrayRef<FilelistInfo> getFilelistInfos() const { return FilelistFileInfos; }
+  ArrayRef<const char *> getArgumentsForTaskExecution() const;
 
   ArrayRef<const Job *> getInputs() const { return Inputs; }
   const CommandOutput &getOutput() const { return *Output; }
@@ -347,6 +361,10 @@ class Job {
 
   static void printArguments(raw_ostream &Stream,
                              const llvm::opt::ArgStringList &Args);
+
+  bool hasResponseFile() const { return ResponseFilePath != nullptr; }
+
+  bool writeArgsToResponseFile() const;
 };
 
 /// A BatchJob comprises a _set_ of jobs, each of which is sufficiently similar

include/swift/Driver/ToolChain.h

@@ -122,6 +122,12 @@ class ToolChain {
     std::vector<std::pair<const char *, const char *>> ExtraEnvironment;
     std::vector<FilelistInfo> FilelistInfos;
 
+    // Not all platforms and jobs support the use of response files, so assume
+    // "false" by default. If the executable specified in the InvocationInfo
+    // constructor supports response files, this can be overridden and set to
+    // "true".
+    bool allowsResponseFiles = false;
+
     InvocationInfo(const char *name, llvm::opt::ArgStringList args = {},
                    decltype(ExtraEnvironment) extraEnv = {})
         : ExecutableName(name), Arguments(std::move(args)),

lib/Driver/Compilation.cpp

@@ -291,8 +291,8 @@ namespace driver {
              "not implemented for compilations with multiple jobs");
       if (Comp.ShowJobLifecycle)
         llvm::outs() << "Added to TaskQueue: " << LogJob(Cmd) << "\n";
-      TQ->addTask(Cmd->getExecutable(), Cmd->getArguments(), llvm::None,
-                  (void *)Cmd);
+      TQ->addTask(Cmd->getExecutable(), Cmd->getArgumentsForTaskExecution(),
+                  llvm::None, (void *)Cmd);
     }
 
     /// When a task finishes, check other Jobs that may be blocked.

lib/Driver/Job.cpp

@@ -16,6 +16,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/raw_ostream.h"
@@ -337,6 +338,15 @@ void Job::dump() const {
   printCommandLineAndEnvironment(llvm::errs());
 }
 
+ArrayRef<const char *> Job::getArgumentsForTaskExecution() const {
+  if (hasResponseFile()) {
+    writeArgsToResponseFile();
+    return getResponseFileArg();
+  } else {
+    return getArguments();
+  }
+}
+
 void Job::printCommandLineAndEnvironment(raw_ostream &Stream,
                                          StringRef Terminator) const {
   printCommandLine(Stream, /*Terminator=*/"");
@@ -352,7 +362,12 @@ void Job::printCommandLineAndEnvironment(raw_ostream &Stream,
 void Job::printCommandLine(raw_ostream &os, StringRef Terminator) const {
   escapeAndPrintString(os, Executable);
   os << ' ';
+  if (hasResponseFile()) {
+    printArguments(os, {ResponseFileArg});
+    os << " # ";
+  }
   printArguments(os, Arguments);
+
   os << Terminator;
 }
 
@@ -403,6 +418,21 @@ void Job::printSummary(raw_ostream &os) const {
   os << "}";
 }
 
+bool Job::writeArgsToResponseFile() const {
+  std::error_code EC;
+  llvm::raw_fd_ostream OS(ResponseFilePath, EC, llvm::sys::fs::F_None);
+  if (EC) {
+    return true;
+  }
+  for (const char *arg : Arguments) {
+    OS << "\"";
+    escapeAndPrintString(OS, arg);
+    OS << "\" ";
+  }
+  OS.flush();
+  return false;
+}
+
 BatchJob::BatchJob(const JobAction &Source,
                    SmallVectorImpl<const Job *> &&Inputs,
                    std::unique_ptr<CommandOutput> Output,

lib/Driver/ToolChain.cpp

@@ -114,11 +114,22 @@ std::unique_ptr<Job> ToolChain::constructJob(
     }
   }
 
+  const char *responseFilePath = nullptr;
+  const char *responseFileArg = nullptr;
+  if (invocationInfo.allowsResponseFiles &&
+      !llvm::sys::commandLineFitsWithinSystemLimits(
+          executablePath, invocationInfo.Arguments)) {
+    responseFilePath = context.getTemporaryFilePath("arguments", "resp");
+    responseFileArg = C.getArgs().MakeArgString(Twine("@") + responseFilePath);
+  }
+
   return llvm::make_unique<Job>(JA, std::move(inputs), std::move(output),
                                 executablePath,
                                 std::move(invocationInfo.Arguments),
                                 std::move(invocationInfo.ExtraEnvironment),
-                                std::move(invocationInfo.FilelistInfos));
+                                std::move(invocationInfo.FilelistInfos),
+                                responseFilePath,
+                                responseFileArg);
 }
 
 std::string

lib/Driver/ToolChains.cpp

@@ -248,6 +248,7 @@ ToolChain::constructInvocation(const CompileJobAction &job,
                                const JobContext &context) const {
   InvocationInfo II{SWIFT_EXECUTABLE_NAME};
   ArgStringList &Arguments = II.Arguments;
+  II.allowsResponseFiles = true;
 
   Arguments.push_back("-frontend");
 

lib/Driver/UnixToolChains.cpp

@@ -325,7 +325,10 @@ toolchains::GenericUnix::constructInvocation(const LinkJobAction &job,
   Arguments.push_back(
       context.Args.MakeArgString(context.Output.getPrimaryOutputFilename()));
 
-  return {Clang, Arguments};
+  InvocationInfo II{Clang, Arguments};
+  II.allowsResponseFiles = true;
+
+  return II;
 }
 
 std::string toolchains::Android::getTargetForLinker() const {

lib/Driver/WindowsToolChains.cpp

@@ -171,5 +171,8 @@ toolchains::Windows::constructInvocation(const LinkJobAction &job,
   Arguments.push_back(
       context.Args.MakeArgString(context.Output.getPrimaryOutputFilename()));
 
-  return {Clang, Arguments};
+  InvocationInfo II{Clang, Arguments};
+  II.allowsResponseFiles = true;
+
+  return II;
 }

test/Driver/response-file.swift

@@ -6,24 +6,35 @@
 // RUN: %target-build-swift -typecheck @%t.1.resp %s 2>&1 | %FileCheck %s -check-prefix=MULTIPLE
 // MULTIPLE: warning: result of call to 'abs' is unused
 
-// RUN: echo "-DTEST2A -DTEST2B" > %t.2.resp
-// RUN: echo "-DTEST2C -DTEST2D" > %t.3.resp
-// RUN: %target-build-swift -typecheck @%t.2.resp %s @%t.3.resp 2>&1 | %FileCheck %s -check-prefix=MIXED
+// RUN: echo "-DTEST2A -DTEST2B" > %t.2A.resp
+// RUN: echo "-DTEST2C -DTEST2D" > %t.2B.resp
+// RUN: %target-build-swift -typecheck @%t.2A.resp %s @%t.2B.resp 2>&1 | %FileCheck %s -check-prefix=MIXED
 // MIXED: warning: result of call to 'abs' is unused
 
-// RUN: echo "-DTEST3A" > %t.4.resp
-// RUN: echo "%s" >> %t.4.resp
-// RUN: echo "-DTEST3B" >> %t.4.resp
-// RUN: %target-build-swift -typecheck @%t.4.resp 2>&1 | %FileCheck %s -check-prefix=RESPONLY
+// RUN: echo "-DTEST3A" > %t.3.resp
+// RUN: echo "%s" >> %t.3.resp
+// RUN: echo "-DTEST3B" >> %t.3.resp
+// RUN: %target-build-swift -typecheck @%t.3.resp 2>&1 | %FileCheck %s -check-prefix=RESPONLY
 // RESPONLY: warning: result of call to 'abs' is unused
 
-// RUN: echo "-DTEST4A" > %t.5.resp
-// RUN: echo "%s" >> %t.5.resp
-// RUN: echo "@%t.5.resp" > %t.6.resp
-// RUN: echo "-DTEST4B" >> %t.6.resp
-// RUN: %target-build-swift -typecheck @%t.6.resp 2>&1 | %FileCheck %s -check-prefix=RECURSIVE
+// RUN: echo "-DTEST4A" > %t.4A.resp
+// RUN: echo "%s" >> %t.4A.resp
+// RUN: echo "@%t.4A.resp" > %t.4B.resp
+// RUN: echo "-DTEST4B" >> %t.4B.resp
+// RUN: %target-build-swift -typecheck @%t.4B.resp 2>&1 | %FileCheck %s -check-prefix=RECURSIVE
 // RECURSIVE: warning: result of call to 'abs' is unused
 
+// RUN: python -c 'for i in range(500001): print "-DTEST5_" + str(i)' > %t.5.resp
+// RUN: %target-build-swift -typecheck @%t.5.resp %s 2>&1 | %FileCheck %s -check-prefix=LONG
+// LONG: warning: result of call to 'abs' is unused
+// RUN: %empty-directory(%t/tmp)
+// RUN: env TMPDIR=%t/tmp/ %target-swiftc_driver %s @%t.5.resp -save-temps 2>&1
+// RUN: ls %t/tmp/arguments-*.resp
+// RUN: %target-build-swift -v @%t.5.resp %s 2>&1 | %FileCheck %s -check-prefix=VERBOSE
+// VERBOSE: @{{[^ ]*}}arguments-{{[0-9a-zA-Z]+}}.resp # -frontend -c -primary-file
+// RUN: not %target-swiftc_driver %s @%t.5.resp -Xfrontend -debug-crash-immediately 2>&1 | %FileCheck %s -check-prefix=CRASH
+// CRASH: Program arguments: {{[^ ]*}}swift -frontend -c -primary-file
+
 #if TEST0
 abs(-5)
 #endif
@@ -43,3 +54,7 @@ abs(-5)
 #if TEST4A && TEST4B
 abs(-5)
 #endif
+
+#if TEST5_0 && TEST5_10 && TEST5_100 && TEST5_1000 && TEST5_10000 && TEST5_100000 && TEST5_500000
+abs(-5)
+#endif