Add optional -emit-symbol-graph output when emitting modules

rdar://71497047

Author: Ashley Garland

include/swift/AST/Decl.h

@@ -866,7 +866,7 @@ class alignas(1 << DeclAlignInBits) Decl {
   }
 
   /// \returns the unparsed comment attached to this declaration.
-  RawComment getRawComment(bool SerializedOK = false) const;
+  RawComment getRawComment(bool SerializedOK = true) const;
 
   Optional<StringRef> getGroupName() const;
 

include/swift/AST/DiagnosticsFrontend.def

@@ -123,6 +123,8 @@ ERROR(error_mode_cannot_emit_interface,none,
       "this mode does not support emitting module interface files", ())
 ERROR(error_mode_cannot_emit_module_summary,none,
       "this mode does not support emitting module summary files", ())
+ERROR(error_mode_cannot_emit_symbol_graph,none,
+      "this mode does not support emitting symbol graph files", ())
 ERROR(cannot_emit_ir_skipping_function_bodies,none,
       "the -experimental-skip-*-function-bodies* flags do not support "
       "emitting IR", ())

include/swift/Frontend/FrontendOptions.h

@@ -373,6 +373,19 @@ class FrontendOptions {
 
   /// Whether we're configured to track system intermodule dependencies.
   bool shouldTrackSystemDependencies() const;
+  
+  /// Whether to emit symbol graphs for the output module.
+  bool EmitSymbolGraph = false;
+
+  /// The directory to which we should emit a symbol graph JSON files.
+  /// It is valid whenever there are any inputs.
+  ///
+  /// These are JSON file that describes the public interface of a module for
+  /// curating documentation, separated into files for each module this module
+  /// extends.
+  ///
+  /// \sa SymbolGraphASTWalker
+  std::string SymbolGraphOutputDir;
 
 private:
   static bool canActionEmitDependencies(ActionType);

include/swift/Option/Options.td

@@ -1159,4 +1159,15 @@ def disable_autolinking_runtime_compatibility_dynamic_replacements
       HelpText<"Do not use autolinking for the dynamic replacement runtime "
                "compatibility library">;
 
+def emit_symbol_graph: Flag<["-"], "emit-symbol-graph">,
+  Flags<[FrontendOption, NoInteractiveOption, DoesNotAffectIncrementalBuild,
+         SupplementaryOutput, HelpHidden]>,
+  HelpText<"Emit a symbol graph">;
+
+def emit_symbol_graph_dir : Separate<["-"], "emit-symbol-graph-dir">,
+  Flags<[FrontendOption, NoInteractiveOption, DoesNotAffectIncrementalBuild,
+         ArgumentIsPath, SupplementaryOutput, HelpHidden]>,
+  HelpText<"Emit a symbol graph to directory <dir>">,
+  MetaVarName<"<dir>">;
+
 include "FrontendOptions.td"

include/swift/Serialization/SerializationOptions.h

@@ -30,6 +30,7 @@ namespace swift {
     const char *OutputPath = nullptr;
     const char *DocOutputPath = nullptr;
     const char *SourceInfoOutputPath = nullptr;
+    std::string SymbolGraphOutputDir;
 
     StringRef GroupInfoPath;
     StringRef ImportedHeader;

lib/AST/Module.cpp

@@ -878,7 +878,7 @@ SourceFile::getBasicLocsForDecl(const Decl *D) const {
   BasicDeclLocs Result;
   Result.SourceFilePath = SM.getDisplayNameForLoc(D->getLoc());
 
-  for (const auto &SRC : D->getRawComment().Comments) {
+  for (const auto &SRC : D->getRawComment(/*SerializedOK*/false).Comments) {
     Result.DocRanges.push_back(std::make_pair(
       LineColumn { SRC.StartLine, SRC.StartColumn },
       SRC.Range.getByteLength())

lib/Driver/ToolChains.cpp

@@ -556,6 +556,11 @@ ToolChain::constructInvocation(const CompileJobAction &job,
       options::
           OPT_disable_autolinking_runtime_compatibility_dynamic_replacements);
 
+  if (context.OI.CompilerMode == OutputInfo::Mode::SingleCompile) {
+    context.Args.AddLastArg(Arguments, options::OPT_emit_symbol_graph);
+    context.Args.AddLastArg(Arguments, options::OPT_emit_symbol_graph_dir);
+  }
+
   return II;
 }
 
@@ -1039,6 +1044,9 @@ ToolChain::constructInvocation(const MergeModuleJobAction &job,
   addOutputsOfType(Arguments, context.Output, context.Args, file_types::TY_TBD,
                    "-emit-tbd-path");
 
+  context.Args.AddLastArg(Arguments, options::OPT_emit_symbol_graph);
+  context.Args.AddLastArg(Arguments, options::OPT_emit_symbol_graph_dir);
+
   context.Args.AddLastArg(Arguments, options::OPT_import_objc_header);
 
   context.Args.AddLastArg(

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -234,6 +234,12 @@ bool ArgsToFrontendOptionsConverter::convert(
   computeImplicitImportModuleNames(OPT_testable_import_module, /*isTestable=*/true);
   computeLLVMArgs();
 
+  Opts.EmitSymbolGraph |= Args.hasArg(OPT_emit_symbol_graph);
+  
+  if (const Arg *A = Args.getLastArg(OPT_emit_symbol_graph_dir)) {
+    Opts.SymbolGraphOutputDir = A->getValue();
+  }
+
   return false;
 }
 
@@ -581,6 +587,11 @@ bool ArgsToFrontendOptionsConverter::checkUnusedSupplementaryOutputPaths()
     Diags.diagnose(SourceLoc(), diag::error_mode_cannot_emit_module_summary);
     return true;
   }
+  if (!FrontendOptions::canActionEmitModule(Opts.RequestedAction) &&
+      !Opts.SymbolGraphOutputDir.empty()) {
+    Diags.diagnose(SourceLoc(), diag::error_mode_cannot_emit_symbol_graph);
+    return true;
+  }
   return false;
 }
 

lib/Frontend/ArgsToFrontendOutputsConverter.cpp

@@ -536,7 +536,8 @@ createFromTypeToPathMap(const TypeToPathMap *map) {
        paths.ModuleInterfaceOutputPath},
       {file_types::TY_SwiftModuleSummaryFile, paths.ModuleSummaryOutputPath},
       {file_types::TY_PrivateSwiftModuleInterfaceFile,
-       paths.PrivateModuleInterfaceOutputPath}};
+       paths.PrivateModuleInterfaceOutputPath},
+  };
   for (const std::pair<file_types::ID, std::string &> &typeAndString :
        typesAndStrings) {
     auto const out = map->find(typeAndString.first);
@@ -559,7 +560,8 @@ SupplementaryOutputPathsComputer::readSupplementaryOutputFileMap() const {
         options::OPT_emit_private_module_interface_path,
         options::OPT_emit_module_source_info_path,
         options::OPT_emit_tbd_path,
-        options::OPT_emit_ldadd_cfile_path)) {
+        options::OPT_emit_ldadd_cfile_path,
+        options::OPT_emit_symbol_graph_dir)) {
     Diags.diagnose(SourceLoc(),
                    diag::error_cannot_have_supplementary_outputs,
                    A->getSpelling(), "-supplementary-output-file-map");

lib/Frontend/CompilerInvocation.cpp

@@ -682,6 +682,12 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
     Opts.AttachCommentsToDecls = true;
   }
 
+  // If we are emitting a symbol graph file, configure lexing and parsing to
+  // remember comments.
+  if (FrontendOpts.EmitSymbolGraph) {
+    Opts.AttachCommentsToDecls = true;
+  }
+
   // If we're parsing SIL, access control doesn't make sense to enforce.
   if (Args.hasArg(OPT_parse_sil) ||
       FrontendOpts.InputsAndOutputs.shouldTreatAsSIL()) {

lib/Frontend/Frontend.cpp

@@ -154,6 +154,19 @@ SerializationOptions CompilerInvocation::computeSerializationOptions(
     serializationOpts.ImportedHeader = opts.ImplicitObjCHeaderPath;
   serializationOpts.ModuleLinkName = opts.ModuleLinkName;
   serializationOpts.ExtraClangOptions = getClangImporterOptions().ExtraArgs;
+  
+  if (opts.EmitSymbolGraph) {
+    if (!opts.SymbolGraphOutputDir.empty()) {
+      serializationOpts.SymbolGraphOutputDir = opts.SymbolGraphOutputDir;
+    } else {
+      serializationOpts.SymbolGraphOutputDir = serializationOpts.OutputPath;
+    }
+    SmallString<256> OutputDir(serializationOpts.SymbolGraphOutputDir);
+    llvm::sys::fs::make_absolute(OutputDir);
+    llvm::sys::path::remove_filename(OutputDir);
+    serializationOpts.SymbolGraphOutputDir = OutputDir.str().str();
+  }
+  
   if (!getIRGenOptions().ForceLoadSymbolName.empty())
     serializationOpts.AutolinkForceLoad = true;
 

lib/Serialization/CMakeLists.txt

@@ -16,5 +16,6 @@ add_swift_host_library(swiftSerialization STATIC
   )
 target_link_libraries(swiftSerialization PRIVATE
   swiftClangImporter
-  swiftSIL)
+  swiftSIL
+  swiftSymbolGraphGen)
 

lib/Serialization/Serialization.cpp

@@ -48,6 +48,8 @@
 #include "swift/Serialization/SerializationOptions.h"
 #include "swift/Strings.h"
 #include "clang/AST/DeclTemplate.h"
+#include "swift/SymbolGraphGen/SymbolGraphOptions.h"
+#include "swift/SymbolGraphGen/SymbolGraphGen.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -5544,4 +5546,20 @@ void swift::serialize(ModuleOrSourceFile DC,
       return false;
     });
   }
+
+  if (!options.SymbolGraphOutputDir.empty()) {
+    if (DC.is<ModuleDecl *>()) {
+      auto *M = DC.get<ModuleDecl*>();
+      FrontendStatsTracer tracer(getContext(DC).Stats,
+                                 "Serialization, symbolgraph");
+      symbolgraphgen::SymbolGraphOptions SGOpts {
+        options.SymbolGraphOutputDir,
+        M->getASTContext().LangOpts.Target,
+        /* PrettyPrint */false,
+        AccessLevel::Public,
+        /*EmitSynthesizedMembers*/true,
+      };
+      symbolgraphgen::emitSymbolGraphForModule(M, SGOpts);
+    }
+  }
 }

lib/SymbolGraphGen/SymbolGraph.cpp

@@ -533,11 +533,9 @@ void SymbolGraph::serialize(llvm::json::OStream &OS) {
           case FileUnitKind::Synthesized:
             llvm_unreachable("Unexpected module kind: Synthesized");
             break;
-          case FileUnitKind::Source: {
-            auto Target = MainFile->getASTContext().LangOpts.Target;
-            symbolgraphgen::serialize(Target, OS);
+          case FileUnitKind::Source:
+            symbolgraphgen::serialize(M.getASTContext().LangOpts.Target, OS);
             break;
-          }
           case FileUnitKind::SerializedAST: {
             auto SerializedAST = cast<SerializedASTFile>(MainFile);
             auto Target = llvm::Triple(SerializedAST->getTargetTriple());

lib/SymbolGraphGen/SymbolGraphGen.cpp

@@ -12,6 +12,8 @@
 
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/Path.h"
+#include "swift/AST/ASTContext.h"
+#include "swift/AST/FileSystem.h"
 #include "swift/SymbolGraphGen/SymbolGraphGen.h"
 
 #include "SymbolGraphASTWalker.h"
@@ -49,18 +51,11 @@ int serializeSymbolGraph(SymbolGraph &SG,
   SmallString<1024> OutputPath(Options.OutputDir);
   llvm::sys::path::append(OutputPath, FileName);
 
-  std::error_code Error;
-  llvm::raw_fd_ostream OS(OutputPath, Error, llvm::sys::fs::FA_Write);
-  if (Error) {
-    llvm::errs() << "Couldn't open output file '" << OutputPath
-        << " for writing: "
-        << Error.message() << "\n";
-    return EXIT_FAILURE;
-  }
-
-  llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0);
-  SG.serialize(J);
-  return EXIT_SUCCESS;
+  return withOutputFile(SG.M.getASTContext().Diags, OutputPath, [&](raw_ostream &OS) {
+    llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0);
+    SG.serialize(J);
+    return false;
+  });
 }
 
 } // end anonymous namespace

test/Serialization/comments-hidden.swift

@@ -92,9 +92,9 @@ private class PrivateClass {
 // TESTING: InternalClass Documentation
 // TESTING: Internal Function Documentation
 
-// SOURCE-LOC: comments-hidden.swift:37:15: Func/PublicClass.__UnderscoredPublic RawComment=none BriefComment=none DocCommentAsXML=none
-// SOURCE-LOC: comments-hidden.swift:39:10: Constructor/PublicClass.init RawComment=none BriefComment=none DocCommentAsXML=none
-// SOURCE-LOC: comments-hidden.swift:41:10: Subscript/PublicClass.subscript RawComment=none BriefComment=none DocCommentAsXML=none
-// SOURCE-LOC: comments-hidden.swift:43:10: Constructor/PublicClass.init RawComment=none BriefComment=none DocCommentAsXML=none
-// SOURCE-LOC: comments-hidden.swift:45:10: Subscript/PublicClass.subscript RawComment=none BriefComment=none DocCommentAsXML=none
-// SOURCE-LOC: comments-hidden.swift:52:15: Func/-= RawComment=none BriefComment=none DocCommentAsXML=none
+// SOURCE-LOC: comments-hidden.swift:37:15: Func/PublicClass.__UnderscoredPublic
+// SOURCE-LOC: comments-hidden.swift:39:10: Constructor/PublicClass.init
+// SOURCE-LOC: comments-hidden.swift:41:10: Subscript/PublicClass.subscript
+// SOURCE-LOC: comments-hidden.swift:43:10: Constructor/PublicClass.init
+// SOURCE-LOC: comments-hidden.swift:45:10: Subscript/PublicClass.subscript
+// SOURCE-LOC: comments-hidden.swift:52:15: Func/-=

test/SymbolGraph/EmitWhileBuilding.swift

@@ -0,0 +1,8 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-build-swift %s -module-name EmitWhileBuilding -emit-module -emit-module-path %t/ -emit-symbol-graph -emit-symbol-graph-dir %t/
+// RUN: %FileCheck %s --input-file %t/EmitWhileBuilding.symbols.json
+
+/// Does a foo.
+public func foo() {}
+
+// CHECK: "precise":"s:17EmitWhileBuilding3fooyyF"