Merge pull request #10726 from bitjammer/upstream-index-while-building-swift-4.0-branch

[index/build] Upstream indexing while building changes

Author: bitjammer

include/swift/AST/DiagnosticsFrontend.def

@@ -167,6 +167,17 @@ ERROR(verify_encountered_fatal,none,
 ERROR(error_parse_input_file,none,
   "error parsing input file '%0' (%1)", (StringRef, StringRef))
 
+ERROR(error_write_index_unit,none,
+  "writing index unit file: %0", (StringRef))
+ERROR(error_create_index_dir,none,
+  "creating index directory: %0", (StringRef))
+ERROR(error_write_index_record,none,
+  "writing index record file: %0", (StringRef))
+ERROR(error_index_failed_status_check,none,
+  "failed file status check: %0", (StringRef))
+ERROR(error_index_inputs_more_than_outputs,none,
+  "index output filenames do not match input source files", ())
+
 ERROR(error_formatting_multiple_file_ranges,none,
   "file ranges don't support multiple input files", ())
 

include/swift/ClangImporter/ClangImporterOptions.h

@@ -37,6 +37,9 @@ class ClangImporterOptions {
   /// Equivalent to Clang's -mcpu=.
   std::string TargetCPU;
 
+  /// The path to which we should store indexing data, if any.
+  std::string IndexStorePath;
+
   /// The bridging header or PCH that will be imported.
   std::string BridgingHeader;
 

include/swift/Driver/Types.def

@@ -61,6 +61,10 @@ TYPE("imported-modules", ImportedModules,   "importedmodules", "")
 TYPE("tbd",             TBD,                "tbd",             "")
 TYPE("module-trace",    ModuleTrace,        "trace.json",      "")
 
+// BEGIN APPLE-ONLY OUTPUT TYPES
+TYPE("index-data",      IndexData,          "",                "")
+// END APPLE-ONLY OUTPUT TYPES
+
 // Misc types
 TYPE("pcm",             ClangModuleFile,    "pcm",             "")
 TYPE("pch",             PCH,                "pch",             "")

include/swift/Frontend/FrontendOptions.h

@@ -132,6 +132,12 @@ class FrontendOptions {
   /// The path to collect the group information for the compiled source files.
   std::string GroupInfoPath;
 
+  /// The path to which we should store indexing data, if any.
+  std::string IndexStorePath;
+
+  /// Emit index data for imported serialized swift system modules.
+  bool IndexSystemModules = false;
+
   /// If non-zero, warn when a function body takes longer than this many
   /// milliseconds to type-check.
   ///

include/swift/Index/IndexRecord.h

@@ -0,0 +1,83 @@
+//===--- IndexRecord.h - Entry point for recording index data ---*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_INDEX_INDEXRECORD_H
+#define SWIFT_INDEX_INDEXRECORD_H
+
+#include "swift/Basic/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace swift {
+class DependencyTracker;
+class ModuleDecl;
+class SourceFile;
+
+namespace index {
+
+/// Index the given source file and store the results to \p indexStorePath.
+///
+/// \param primarySourceFile The source file to index.
+///
+/// \param indexUnitToken A unique identifier for this translation unit in the
+/// form of a file path.
+///
+/// \param indexStorePath The location to write the indexing data to.
+///
+/// \param indexSystemModules If true, emit index data for imported serialized
+/// swift system modules.
+///
+/// \param isDebugCompilation true for non-optimized compiler invocation.
+///
+/// \param targetTriple The target for this compilation.
+///
+/// \param dependencyTracker The set of dependencies seen while building.
+bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
+                    StringRef indexStorePath, bool indexSystemModules,
+                    bool isDebugCompilation, StringRef targetTriple,
+                    const DependencyTracker &dependencyTracker);
+
+/// Index the given module and store the results to \p indexStorePath.
+///
+/// \param module The module to index.
+///
+/// \param indexUnitTokens A list of unique identifiers for the index units to
+/// be written. This may either be one unit per source file of \p module, or it
+/// may be a single unit, in which case all the index information will be
+/// combined into a single unit.
+///
+/// \param moduleUnitToken A unique identifier for this module unit in the form
+/// of a file path. Only used if \p indexUnitTokens are specified for each
+/// source file, otherwise the single \p indexUnitTokens value is used instead.
+///
+/// \param indexStorePath The location to write the indexing data to.
+///
+/// \param indexSystemModules If true, emit index data for imported serialized
+/// swift system modules.
+///
+/// \param isDebugCompilation true for non-optimized compiler invocation.
+///
+/// \param targetTriple The target for this compilation.
+///
+/// \param dependencyTracker The set of dependencies seen while building.
+bool indexAndRecord(ModuleDecl *module, ArrayRef<std::string> indexUnitTokens,
+                    StringRef moduleUnitToken, StringRef indexStorePath,
+                    bool indexSystemModules, bool isDebugCompilation,
+                    StringRef targetTriple,
+                    const DependencyTracker &dependencyTracker);
+// FIXME: indexUnitTokens could be StringRef, but that creates an impedance
+// mismatch in the caller.
+
+} // end namespace index
+} // end namespace swift
+
+#endif // SWIFT_INDEX_INDEXRECORD_H

include/swift/Option/FrontendOptions.td

@@ -414,6 +414,9 @@ def external_pass_pipeline_filename : Separate<["-"], "external-pass-pipeline-fi
     HelpText<"Use the pass pipeline defined by <pass_pipeline_file>">,
     MetaVarName<"<pass_pipeline_file>">;
 
+def index_system_modules : Flag<["-"], "index-system-modules">,
+  HelpText<"Emit index data for imported serialized swift system modules">;
+
 def dump_interface_hash : Flag<["-"], "dump-interface-hash">,
    HelpText<"Parse input file(s) and dump interface token hash(es)">,
    ModeOpt;

include/swift/Option/Options.td

@@ -646,6 +646,18 @@ def sanitize_coverage_EQ : CommaJoined<["-"], "sanitize-coverage=">,
   HelpText<"Specify the type of coverage instrumentation for Sanitizers and"
   " additional options separated by commas">;
 
+def index_file : Flag<["-"], "index-file">,
+  HelpText<"Produce index data for a source file">, ModeOpt,
+  Flags<[NoInteractiveOption, DoesNotAffectIncrementalBuild]>;
+def index_file_path : Separate<["-"], "index-file-path">,
+  Flags<[NoInteractiveOption, DoesNotAffectIncrementalBuild]>,
+  HelpText<"Produce index data for file <path>">,
+  MetaVarName<"<path>">;
+
+def index_store_path : Separate<["-"], "index-store-path">,
+  Flags<[FrontendOption]>, MetaVarName<"<path>">,
+  HelpText<"Store indexing data to <path>">;
+
 def enforce_exclusivity_EQ : Joined<["-"], "enforce-exclusivity=">,
   Flags<[FrontendOption]>, MetaVarName<"<enforcement>">,
   HelpText<"Enforce law of exclusivity">;

lib/ClangImporter/ClangImporter.cpp

@@ -46,6 +46,7 @@
 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/Utils.h"
+#include "clang/Index/IndexingAction.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/Lex/Preprocessor.h"
@@ -705,6 +706,11 @@ addCommonInvocationArguments(std::vector<std::string> &invocationArgStrs,
     invocationArgStrs.push_back(overrideResourceDir);
   }
 
+  if (!importerOpts.IndexStorePath.empty()) {
+    invocationArgStrs.push_back("-index-store-path");
+    invocationArgStrs.push_back(importerOpts.IndexStorePath);
+  }
+
   for (auto extraArg : importerOpts.ExtraArgs) {
     invocationArgStrs.push_back(extraArg);
   }
@@ -1306,6 +1312,7 @@ ClangImporter::emitBridgingPCH(StringRef headerPath,
   invocation->getFrontendOpts().Inputs.push_back(
       clang::FrontendInputFile(headerPath, clang::IK_ObjC));
   invocation->getFrontendOpts().OutputFile = outputPCHPath;
+  invocation->getFrontendOpts().ProgramAction = clang::frontend::GeneratePCH;
   invocation->getPreprocessorOpts().resetNonModularOptions();
 
   clang::CompilerInstance emitInstance(
@@ -1319,8 +1326,15 @@ ClangImporter::emitBridgingPCH(StringRef headerPath,
   emitInstance.createSourceManager(fileManager);
   emitInstance.setTarget(&Impl.Instance->getTarget());
 
-  clang::GeneratePCHAction action;
-  emitInstance.ExecuteAction(action);
+  std::unique_ptr<clang::FrontendAction> action;
+  action.reset(new clang::GeneratePCHAction());
+  if (!emitInstance.getFrontendOpts().IndexStorePath.empty()) {
+    action = clang::index::
+      createIndexDataRecordingAction(emitInstance.getFrontendOpts(),
+                                     std::move(action));
+  }
+  emitInstance.ExecuteAction(*action);
+
   if (emitInstance.getDiagnostics().hasErrorOccurred()) {
     Impl.SwiftContext.Diags.diagnose({},
                                      diag::bridging_header_pch_error,
@@ -1407,6 +1421,17 @@ ModuleDecl *ClangImporter::loadModule(
     auto importRAII = diagClient.handleImport(clangPath.front().first,
                                               importLoc);
 
+    std::string preservedIndexStorePathOption;
+    auto &clangFEOpts = Impl.Instance->getFrontendOpts();
+    if (!clangFEOpts.IndexStorePath.empty()) {
+      StringRef moduleName = path[0].first->getName();
+      // Ignore the SwiftShims module for the index data.
+      if (moduleName == Impl.SwiftContext.SwiftShimsModuleName.str()) {
+        preservedIndexStorePathOption = clangFEOpts.IndexStorePath;
+        clangFEOpts.IndexStorePath.clear();
+      }
+    }
+
     // FIXME: The source location here is completely bogus. It can't be
     // invalid, it can't be the same thing twice in a row, and it has to come
     // from an actual buffer, so we make a fake buffer and just use a counter.
@@ -1427,6 +1452,12 @@ ModuleDecl *ClangImporter::loadModule(
     clang::ModuleLoadResult result =
         Impl.Instance->loadModule(clangImportLoc, path, visibility,
                                   /*IsInclusionDirective=*/false);
+
+    if (!preservedIndexStorePathOption.empty()) {
+      // Restore the -index-store-path option.
+      clangFEOpts.IndexStorePath = preservedIndexStorePathOption;
+    }
+
     if (result && makeVisible)
       Impl.getClangPreprocessor().makeModuleVisible(result, clangImportLoc);
     return result;

lib/Driver/Driver.cpp

@@ -1128,6 +1128,17 @@ void Driver::buildOutputInfo(const ToolChain &TC, const DerivedArgList &Args,
       OI.CompilerMode = OutputInfo::Mode::SingleCompile;
       break;
 
+    // BEGIN APPLE-ONLY OUTPUT ACTIONS
+    case options::OPT_index_file:
+      OI.CompilerMode = OutputInfo::Mode::SingleCompile;
+      OI.CompilerOutputType = types::TY_IndexData;
+      break;
+    // END APPLE-ONLY OUTPUT ACTIONS
+
+    case options::OPT_update_code:
+      OI.CompilerOutputType = types::TY_Remapping;
+      OI.LinkAction = LinkKind::None;
+      break;
     case options::OPT_parse:
     case options::OPT_typecheck:
     case options::OPT_dump_parse:
@@ -1420,6 +1431,7 @@ void Driver::buildActions(const ToolChain &TC,
       case types::TY_ClangModuleFile:
       case types::TY_SwiftDeps:
       case types::TY_Remapping:
+      case types::TY_IndexData:
       case types::TY_PCH:
       case types::TY_ImportedModules:
       case types::TY_TBD:

lib/Driver/ToolChains.cpp

@@ -231,6 +231,13 @@ ToolChain::constructInvocation(const CompileJobAction &job,
     case types::TY_TBD:
       FrontendModeOption = "-emit-tbd";
       break;
+
+    // BEGIN APPLE-ONLY OUTPUT TYPES
+    case types::TY_IndexData:
+      FrontendModeOption = "-typecheck";
+      break;
+    // END APPLE-ONLY OUTPUT TYPES
+
     case types::TY_Remapping:
       FrontendModeOption = "-update-code";
       break;
@@ -311,6 +318,12 @@ ToolChain::constructInvocation(const CompileJobAction &job,
     break;
   }
   case OutputInfo::Mode::SingleCompile: {
+    if (context.Output.getPrimaryOutputType() == types::TY_IndexData) {
+      if (Arg *A = context.Args.getLastArg(options::OPT_index_file_path)) {
+        Arguments.push_back("-primary-file");
+        Arguments.push_back(A->getValue());
+      }
+    }
     if (context.Args.hasArg(options::OPT_driver_use_filelists) ||
         context.InputActions.size() > TOO_MANY_FILES) {
       Arguments.push_back("-filelist");
@@ -456,6 +469,11 @@ ToolChain::constructInvocation(const CompileJobAction &job,
   if (context.Args.hasArg(options::OPT_embed_bitcode_marker))
     Arguments.push_back("-embed-bitcode-marker");
 
+  if (context.Args.hasArg(options::OPT_index_store_path)) {
+    context.Args.AddLastArg(Arguments, options::OPT_index_store_path);
+    Arguments.push_back("-index-system-modules");
+  }
+
   return II;
 }
 
@@ -541,6 +559,7 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case types::TY_SIL:
     case types::TY_SIB:
     case types::TY_PCH:
+    case types::TY_IndexData:
       llvm_unreachable("Cannot be output from backend job");
     case types::TY_Swift:
     case types::TY_dSYM:
@@ -824,6 +843,7 @@ ToolChain::constructInvocation(const GeneratePCHJobAction &job,
                         Arguments);
 
   addInputsOfType(Arguments, context.InputActions, types::TY_ObjCHeader);
+  context.Args.AddLastArg(Arguments, options::OPT_index_store_path);
 
   if (job.isPersistentPCH()) {
     Arguments.push_back("-emit-pch");

lib/Driver/Types.cpp

@@ -92,6 +92,7 @@ bool types::isTextual(ID Id) {
   case types::TY_SwiftDeps:
   case types::TY_Nothing:
   case types::TY_Remapping:
+  case types::TY_IndexData:
     return false;
   case types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");
@@ -128,6 +129,7 @@ bool types::isAfterLLVM(ID Id) {
   case types::TY_SwiftDeps:
   case types::TY_Nothing:
   case types::TY_Remapping:
+  case types::TY_IndexData:
   case types::TY_ModuleTrace:
     return false;
   case types::TY_INVALID:
@@ -165,6 +167,7 @@ bool types::isPartOfSwiftCompilation(ID Id) {
   case types::TY_SwiftDeps:
   case types::TY_Nothing:
   case types::TY_Remapping:
+  case types::TY_IndexData:
   case types::TY_ModuleTrace:
     return false;
   case types::TY_INVALID:

lib/Frontend/CompilerInvocation.cpp

@@ -169,6 +169,11 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     Opts.GroupInfoPath = A->getValue();
   }
 
+  if (const Arg *A = Args.getLastArg(OPT_index_store_path)) {
+    Opts.IndexStorePath = A->getValue();
+  }
+  Opts.IndexSystemModules |= Args.hasArg(OPT_index_system_modules);
+
   Opts.EmitVerboseSIL |= Args.hasArg(OPT_emit_verbose_sil);
   Opts.EmitSortedSIL |= Args.hasArg(OPT_emit_sorted_sil);
 
@@ -1143,6 +1148,9 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts,
   if (const Arg *A = Args.getLastArg(OPT_target_cpu))
     Opts.TargetCPU = A->getValue();
 
+  if (const Arg *A = Args.getLastArg(OPT_index_store_path))
+    Opts.IndexStorePath = A->getValue();
+
   for (const Arg *A : make_range(Args.filtered_begin(OPT_Xcc),
                                  Args.filtered_end())) {
     Opts.ExtraArgs.push_back(A->getValue());

lib/Frontend/Frontend.cpp

@@ -97,6 +97,12 @@ bool CompilerInstance::setup(const CompilerInvocation &Invok) {
   if (!Invocation.getFrontendOptions().ModuleDocOutputPath.empty())
     Invocation.getLangOptions().AttachCommentsToDecls = true;
 
+  // If we are doing index-while-building, configure lexing and parsing to
+  // remember comments.
+  if (!Invocation.getFrontendOptions().IndexStorePath.empty()) {
+    Invocation.getLangOptions().AttachCommentsToDecls = true;
+  }
+
   Context.reset(new ASTContext(Invocation.getLangOptions(),
                                Invocation.getSearchPathOptions(),
                                SourceMgr, Diagnostics));

lib/FrontendTool/CMakeLists.txt

@@ -5,6 +5,7 @@ add_swift_library(swiftFrontendTool STATIC
   TBD.cpp
   DEPENDS SwiftOptions
   LINK_LIBRARIES
+    swiftIndex
     swiftIDE
     swiftTBDGen swiftIRGen swiftSIL swiftSILGen swiftSILOptimizer
     swiftDemangling