[Frontend] Add a new -index-unit-ouput-path and filelist equivalent to the frontend

These new options mirror -o and -output-filelist and are used instead
of those options to supply the output file path(s) to record in the
index store. This is intended to allow sharing index data across
builds in separate directories that are otherwise equivalent as far
as the index data is concered (e.g. an ASAN build and a non-ASAN build)
by supplying the same -index-unit-output-path for both.

Resolves rdar://problem/74816412

Author: Nathan Hawes

include/swift/AST/DiagnosticsFrontend.def

@@ -165,15 +165,15 @@ ERROR(error_mode_requires_one_sil_multi_sib,none,
       ())
 
 ERROR(error_no_output_filename_specified,none,
-      "an output filename was not specified for a mode which requires an "
-      "output filename", ())
+      "an %0 filename was not specified for a mode which requires an "
+      "%0 filename", (StringRef))
 
 ERROR(error_implicit_output_file_is_directory,none,
       "the implicit output file '%0' is a directory; explicitly specify a "
-      "filename using -o", (StringRef))
+      "filename using %1", (StringRef, StringRef))
 
 ERROR(error_if_any_output_files_are_specified_they_all_must_be,none,
-      "if any output files are specified, they all must be", ())
+      "if any %0 files are specified, they all must be", (StringRef))
 
 ERROR(error_primary_file_not_found,none,
       "primary file '%0' was not found in file list '%1'",
@@ -211,6 +211,8 @@ ERROR(error_index_failed_status_check,none,
       "failed file status check: %0", (StringRef))
 ERROR(error_index_inputs_more_than_outputs,none,
       "index output filenames do not match input source files", ())
+WARNING(warn_index_unit_output_path_without_index_store,none,
+      "-index-unit-output-path is ignored without -index-store-path", ())
 
 ERROR(error_wrong_number_of_arguments,none,
       "wrong number of '%0' arguments (expected %1, got %2)",

include/swift/Basic/PrimarySpecificPaths.h

@@ -32,16 +32,23 @@ class PrimarySpecificPaths {
   /// is to be written to stdout, contains "-".
   std::string OutputFilename;
 
+  /// The name to report the main output file as being in the index store.
+  /// This is equivalent to OutputFilename, unless -index-store-output-path
+  /// was specified.
+  std::string IndexUnitOutputFilename;
+
   SupplementaryOutputPaths SupplementaryOutputs;
 
   /// The name of the "main" input file, used by the debug info.
   std::string MainInputFilenameForDebugInfo;
 
   PrimarySpecificPaths(StringRef OutputFilename = StringRef(),
+                       StringRef IndexUnitOutputFilename = StringRef(),
                        StringRef MainInputFilenameForDebugInfo = StringRef(),
                        SupplementaryOutputPaths SupplementaryOutputs =
                            SupplementaryOutputPaths())
       : OutputFilename(OutputFilename),
+        IndexUnitOutputFilename(IndexUnitOutputFilename),
         SupplementaryOutputs(SupplementaryOutputs),
         MainInputFilenameForDebugInfo(MainInputFilenameForDebugInfo) {}
 

include/swift/Basic/SupplementaryOutputPaths.h

@@ -185,7 +185,7 @@ struct SupplementaryOutputPaths {
     if (!LdAddCFilePath.empty())
       fn(LdAddCFilePath); 
     if (!ModuleSummaryOutputPath.empty())
-      fn(ModuleSummaryOutputPath); 
+      fn(ModuleSummaryOutputPath);
   }
 
   bool empty() const {

include/swift/Frontend/FrontendInputsAndOutputs.h

@@ -188,7 +188,8 @@ class FrontendInputsAndOutputs {
   friend struct InterfaceSubContextDelegateImpl;
   void setMainAndSupplementaryOutputs(
       ArrayRef<std::string> outputFiles,
-      ArrayRef<SupplementaryOutputPaths> supplementaryOutputs);
+      ArrayRef<SupplementaryOutputPaths> supplementaryOutputs,
+      ArrayRef<std::string> outputFilesForIndexUnits = None);
 
 public:
   unsigned countOfInputsProducingMainOutputs() const;
@@ -209,13 +210,18 @@ class FrontendInputsAndOutputs {
       llvm::function_ref<bool(const InputFile &)> fn) const;
 
   std::vector<std::string> copyOutputFilenames() const;
+  std::vector<std::string> copyIndexUnitOutputFilenames() const;
 
   void forEachOutputFilename(llvm::function_ref<void(StringRef)> fn) const;
 
   /// Gets the name of the specified output filename.
   /// If multiple files are specified, the last one is returned.
   std::string getSingleOutputFilename() const;
 
+  /// Gets the name of the specified output filename to record in the index unit
+  /// output files. If multiple are specified, the last one is returned.
+  std::string getSingleIndexUnitOutputFilename() const;
+
   bool isOutputFilenameStdout() const;
   bool isOutputFileDirectory() const;
   bool hasNamedOutputFile() const;

include/swift/Frontend/InputFile.h

@@ -105,6 +105,12 @@ class InputFile final {
   /// stream, the result is "-".
   std::string outputFilename() const { return PSPs.OutputFilename; }
 
+  std::string indexUnitOutputFilename() const {
+    if (!PSPs.IndexUnitOutputFilename.empty())
+      return PSPs.IndexUnitOutputFilename;
+    return outputFilename();
+  }
+
   /// If there are explicit primary inputs (i.e. designated with -primary-input
   /// or -primary-filelist), the paths specific to those inputs (other than the
   /// input file path itself) are kept here. If there are no explicit primary

include/swift/Option/FrontendOptions.td

@@ -626,6 +626,9 @@ def index_ignore_stdlib :
   Flag<["-"], "index-ignore-stdlib">,
   HelpText<"Avoid emitting index data for the standard library.">;
 
+def index_unit_output_path_filelist : Separate<["-"], "index-unit-output-path-filelist">,
+  HelpText<"Specify index unit output paths in a file rather than on the command line">;
+
 def dump_interface_hash : Flag<["-"], "dump-interface-hash">,
    HelpText<"Parse input file(s) and dump interface token hash(es)">,
    ModeOpt;

include/swift/Option/Options.td

@@ -1123,6 +1123,10 @@ def index_store_path : Separate<["-"], "index-store-path">,
   Flags<[FrontendOption, ArgumentIsPath]>, MetaVarName<"<path>">,
   HelpText<"Store indexing data to <path>">;
 
+def index_unit_output_path : Separate<["-"], "index-unit-output-path">,
+  Flags<[FrontendOption, ArgumentIsPath]>, MetaVarName<"<path>">,
+  HelpText<"Use <path> as the output path in the produced index data.">;
+
 def index_ignore_system_modules : Flag<["-"], "index-ignore-system-modules">,
   Flags<[NoInteractiveOption]>,
   HelpText<"Avoid indexing system modules">;

lib/Frontend/ArgsToFrontendInputsConverter.cpp

@@ -187,8 +187,8 @@ ArgsToFrontendInputsConverter::createInputFilesConsumingPrimaries(
 
   if (!Files.empty() && !hasAnyPrimaryFiles) {
     Optional<std::vector<std::string>> userSuppliedNamesOrErr =
-        OutputFilesComputer::getOutputFilenamesFromCommandLineOrFilelist(Args,
-                                                                         Diags);
+        OutputFilesComputer::getOutputFilenamesFromCommandLineOrFilelist(
+          Args, Diags, options::OPT_o, options::OPT_output_filelist);
     if (userSuppliedNamesOrErr && userSuppliedNamesOrErr->size() == 1)
       result.setIsSingleThreadedWMO(true);
   }

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -514,8 +514,8 @@ bool ArgsToFrontendOptionsConverter::computeFallbackModuleName() {
     return false;
   }
   Optional<std::vector<std::string>> outputFilenames =
-      OutputFilesComputer::getOutputFilenamesFromCommandLineOrFilelist(Args,
-                                                                       Diags);
+      OutputFilesComputer::getOutputFilenamesFromCommandLineOrFilelist(
+        Args, Diags, options::OPT_o, options::OPT_output_filelist);
 
   std::string nameToStem =
       outputFilenames && outputFilenames->size() == 1 &&
@@ -531,14 +531,17 @@ bool ArgsToFrontendOptionsConverter::computeFallbackModuleName() {
 bool ArgsToFrontendOptionsConverter::
     computeMainAndSupplementaryOutputFilenames() {
   std::vector<std::string> mainOutputs;
+  std::vector<std::string> mainOutputForIndexUnits;
   std::vector<SupplementaryOutputPaths> supplementaryOutputs;
   const bool hadError = ArgsToFrontendOutputsConverter(
                             Args, Opts.ModuleName, Opts.InputsAndOutputs, Diags)
-                            .convert(mainOutputs, supplementaryOutputs);
+                            .convert(mainOutputs, mainOutputForIndexUnits,
+                                     supplementaryOutputs);
   if (hadError)
     return true;
   Opts.InputsAndOutputs.setMainAndSupplementaryOutputs(mainOutputs,
-                                                       supplementaryOutputs);
+                                                       supplementaryOutputs,
+                                                       mainOutputForIndexUnits);
   return false;
 }
 

lib/Frontend/ArgsToFrontendOutputsConverter.cpp

@@ -34,16 +34,43 @@ using namespace llvm::opt;
 
 bool ArgsToFrontendOutputsConverter::convert(
     std::vector<std::string> &mainOutputs,
+    std::vector<std::string> &mainOutputsForIndexUnits,
     std::vector<SupplementaryOutputPaths> &supplementaryOutputs) {
 
   Optional<OutputFilesComputer> ofc =
-      OutputFilesComputer::create(Args, Diags, InputsAndOutputs);
+      OutputFilesComputer::create(Args, Diags, InputsAndOutputs, {
+        "output", options::OPT_o, options::OPT_output_filelist, "-o"
+      });
   if (!ofc)
     return true;
   Optional<std::vector<std::string>> mains = ofc->computeOutputFiles();
   if (!mains)
     return true;
 
+  Optional<std::vector<std::string>> indexMains;
+  if (Args.hasArg(options::OPT_index_unit_output_path,
+                  options::OPT_index_unit_output_path_filelist)) {
+
+    if (!Args.hasArg(options::OPT_index_store_path)) {
+      Diags.diagnose(SourceLoc(),
+                     diag::warn_index_unit_output_path_without_index_store);
+    }
+
+    Optional<OutputFilesComputer> iuofc =
+        OutputFilesComputer::create(Args, Diags, InputsAndOutputs, {
+          "index unit output path", options::OPT_index_unit_output_path,
+          options::OPT_index_unit_output_path_filelist,
+          "-index-unit-output-path"
+        });
+    if (!iuofc)
+      return true;
+    indexMains = iuofc->computeOutputFiles();
+    if (!indexMains)
+      return true;
+
+    assert(mains->size() == indexMains->size() && "checks not equivalent?");
+  }
+
   Optional<std::vector<SupplementaryOutputPaths>> supplementaries =
       SupplementaryOutputPathsComputer(Args, Diags, InputsAndOutputs, *mains,
                                        ModuleName)
@@ -52,6 +79,8 @@ bool ArgsToFrontendOutputsConverter::convert(
     return true;
 
   mainOutputs = std::move(*mains);
+  if (indexMains)
+    mainOutputsForIndexUnits = std::move(*indexMains);
   supplementaryOutputs = std::move(*supplementaries);
   return false;
 }
@@ -75,22 +104,26 @@ ArgsToFrontendOutputsConverter::readOutputFileList(const StringRef filelistPath,
 
 Optional<std::vector<std::string>>
 OutputFilesComputer::getOutputFilenamesFromCommandLineOrFilelist(
-    const ArgList &args, DiagnosticEngine &diags) {
-  if (const Arg *A = args.getLastArg(options::OPT_output_filelist)) {
-    assert(!args.hasArg(options::OPT_o) &&
-           "don't use -o with -output-filelist");
+    const ArgList &args, DiagnosticEngine &diags, options::ID singleOpt,
+    options::ID filelistOpt) {
+  if (const Arg *A = args.getLastArg(filelistOpt)) {
+    assert(!args.hasArg(singleOpt) &&
+           "don't use -o with -output-filelist or -index-unit-output-path with "
+           " -index-unit-output-filelist");
     return ArgsToFrontendOutputsConverter::readOutputFileList(A->getValue(),
                                                               diags);
   }
-  return args.getAllArgValues(options::OPT_o);
+  return args.getAllArgValues(singleOpt);
 }
 
 Optional<OutputFilesComputer>
 OutputFilesComputer::create(const llvm::opt::ArgList &args,
                             DiagnosticEngine &diags,
-                            const FrontendInputsAndOutputs &inputsAndOutputs) {
+                            const FrontendInputsAndOutputs &inputsAndOutputs,
+                            OutputOptInfo optInfo) {
   Optional<std::vector<std::string>> outputArguments =
-      getOutputFilenamesFromCommandLineOrFilelist(args, diags);
+      getOutputFilenamesFromCommandLineOrFilelist(args, diags, optInfo.SingleID,
+                                                  optInfo.FilelistID);
   if (!outputArguments)
     return None;
   const StringRef outputDirectoryArgument =
@@ -113,7 +146,8 @@ OutputFilesComputer::create(const llvm::opt::ArgList &args,
           inputsAndOutputs.countOfInputsProducingMainOutputs()) {
     diags.diagnose(
         SourceLoc(),
-        diag::error_if_any_output_files_are_specified_they_all_must_be);
+        diag::error_if_any_output_files_are_specified_they_all_must_be,
+        optInfo.PrettyName);
     return None;
   }
 
@@ -125,7 +159,8 @@ OutputFilesComputer::create(const llvm::opt::ArgList &args,
       outputDirectoryArgument, firstInput, requestedAction,
       args.getLastArg(options::OPT_module_name),
       file_types::getExtension(outputType),
-      FrontendOptions::doesActionProduceTextualOutput(requestedAction));
+      FrontendOptions::doesActionProduceTextualOutput(requestedAction),
+      optInfo);
 }
 
 OutputFilesComputer::OutputFilesComputer(
@@ -135,12 +170,13 @@ OutputFilesComputer::OutputFilesComputer(
     const StringRef outputDirectoryArgument, const StringRef firstInput,
     const FrontendOptions::ActionType requestedAction,
     const llvm::opt::Arg *moduleNameArg, const StringRef suffix,
-    const bool hasTextualOutput)
+    const bool hasTextualOutput, OutputOptInfo optInfo)
     : Diags(diags), InputsAndOutputs(inputsAndOutputs),
       OutputFileArguments(outputFileArguments),
       OutputDirectoryArgument(outputDirectoryArgument), FirstInput(firstInput),
       RequestedAction(requestedAction), ModuleNameArg(moduleNameArg),
-      Suffix(suffix), HasTextualOutput(hasTextualOutput) {}
+      Suffix(suffix), HasTextualOutput(hasTextualOutput),
+      OutputInfo(optInfo) {}
 
 Optional<std::vector<std::string>>
 OutputFilesComputer::computeOutputFiles() const {
@@ -189,7 +225,8 @@ OutputFilesComputer::deriveOutputFileFromInput(const InputFile &input) const {
   std::string baseName = determineBaseNameOfOutput(input);
   if (baseName.empty()) {
     // Assuming FrontendOptions::doesActionProduceOutput(RequestedAction)
-    Diags.diagnose(SourceLoc(), diag::error_no_output_filename_specified);
+    Diags.diagnose(SourceLoc(), diag::error_no_output_filename_specified,
+                   OutputInfo.PrettyName);
     return None;
   }
   return deriveOutputFileFromParts("", baseName);
@@ -200,7 +237,7 @@ Optional<std::string> OutputFilesComputer::deriveOutputFileForDirectory(
   std::string baseName = determineBaseNameOfOutput(input);
   if (baseName.empty()) {
     Diags.diagnose(SourceLoc(), diag::error_implicit_output_file_is_directory,
-                   OutputDirectoryArgument);
+                   OutputDirectoryArgument, OutputInfo.SingleOptSpelling);
     return None;
   }
   return deriveOutputFileFromParts(OutputDirectoryArgument, baseName);

lib/Frontend/ArgsToFrontendOutputsConverter.h

@@ -44,6 +44,7 @@ class ArgsToFrontendOutputsConverter {
         Diags(diags) {}
 
   bool convert(std::vector<std::string> &mainOutputs,
+               std::vector<std::string> &mainOutputsForIndexUnits,
                std::vector<SupplementaryOutputPaths> &supplementaryOutputs);
 
   /// Try to read an output file list file.
@@ -52,6 +53,13 @@ class ArgsToFrontendOutputsConverter {
   readOutputFileList(StringRef filelistPath, DiagnosticEngine &diags);
 };
 
+struct OutputOptInfo {
+  StringRef PrettyName;
+  options::ID SingleID;
+  options::ID FilelistID;
+  StringRef SingleOptSpelling;
+};
+
 class OutputFilesComputer {
   DiagnosticEngine &Diags;
   const FrontendInputsAndOutputs &InputsAndOutputs;
@@ -62,26 +70,31 @@ class OutputFilesComputer {
   const llvm::opt::Arg *const ModuleNameArg;
   const StringRef Suffix;
   const bool HasTextualOutput;
+  const OutputOptInfo OutputInfo;
 
   OutputFilesComputer(DiagnosticEngine &diags,
                       const FrontendInputsAndOutputs &inputsAndOutputs,
                       std::vector<std::string> outputFileArguments,
                       StringRef outputDirectoryArgument, StringRef firstInput,
                       FrontendOptions::ActionType requestedAction,
                       const llvm::opt::Arg *moduleNameArg, StringRef suffix,
-                      bool hasTextualOutput);
+                      bool hasTextualOutput,
+                      OutputOptInfo optInfo);
 
 public:
   static Optional<OutputFilesComputer>
   create(const llvm::opt::ArgList &args, DiagnosticEngine &diags,
-         const FrontendInputsAndOutputs &inputsAndOutputs);
+         const FrontendInputsAndOutputs &inputsAndOutputs,
+         OutputOptInfo optInfo);
 
   /// \return the output filenames on the command line or in the output
   /// filelist. If there
   /// were neither -o's nor an output filelist, returns an empty vector.
   static Optional<std::vector<std::string>>
   getOutputFilenamesFromCommandLineOrFilelist(const llvm::opt::ArgList &args,
-                                              DiagnosticEngine &diags);
+                                              DiagnosticEngine &diags,
+                                              options::ID singleOpt,
+                                              options::ID filelistOpt);
 
   Optional<std::vector<std::string>> computeOutputFiles() const;