[ModuleTrace] Emit import graph to ease debugging import issues.

This should make it easier to spot cycles in import graphs, as well as
funky behavior involving both default/private imports + SPI imports.

Author: varungandhi-apple

include/swift/AST/Module.h

@@ -667,6 +667,15 @@ class ModuleDecl : public DeclContext, public TypeDecl {
     /// this flag is specified.
     ShadowedBySeparateOverlay = 1 << 4
   };
+
+  static constexpr std::array<ImportFilterKind, 5> AllImportFilterKinds {
+    ImportFilterKind::Public,
+    ImportFilterKind::Private,
+    ImportFilterKind::ImplementationOnly,
+    ImportFilterKind::SPIAccessControl,
+    ImportFilterKind::ShadowedBySeparateOverlay,
+  };
+
   /// \sa getImportedModules
   using ImportFilter = OptionSet<ImportFilterKind>;
 

include/swift/Basic/FileTypes.def

@@ -72,6 +72,8 @@ TYPE("tbd",                 TBD,                       "tbd",             "")
 // Swift section of the internal wiki.
 TYPE("module-trace",        ModuleTrace,               "trace.json",      "")
 
+TYPE("module-import-graph", ModuleImportGraph,         "import-graph.dot","")
+
 // Complete dependency information for the given Swift files as JSON.
 TYPE("json-dependencies",   JSONDependencies,          "dependencies.json",      "")
 

include/swift/Basic/GraphViz.h

@@ -0,0 +1,152 @@
+//===--- GraphViz.h - Utilities for outputting graphs -----------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Defines data structures for creating transient graphs in order to print
+/// them to GraphViz DOT format.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_BASIC_GRAPHVIZ_H
+#define SWIFT_BASIC_GRAPHVIZ_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <iterator>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace swift {
+// MARK: - MultiGraph
+
+/// A multigraph data structure, with a set of edges connecting pairs of nodes.
+///
+/// The fields are left public so that you can potentially customize printing
+/// directly, instead of relying on the existing customization points.
+///
+/// \invariant {
+///   \code forall n1 n2. nodePairs[{n1, n2}] < edgeSets.size() \endcode
+/// }
+template <typename Node, typename Edge, typename EdgeSet>
+struct MultiGraph {
+  /// List of nodes stored in the graph.
+  llvm::DenseMap<std::pair<Node, Node>, size_t> nodePairs;
+
+  /// List of edges stored in the graph.
+  std::vector<EdgeSet> edgeSets;
+
+  /// Three way comparison function for nodes used for deterministic output.
+  llvm::function_ref<int (Node, Node)> compareNodes;
+
+  /// Three way comparison function for edges used for deterministic output.
+  llvm::function_ref<int (Edge, Edge)> compareEdges;
+
+  /// Default attributes used for styling nodes.
+  std::string defaultNodeAttrs =
+      "shape = \"box\", style = \"rounded\", penwidth = \"2\"";
+
+  /// Print the name of a node.
+  llvm::function_ref<void (Node, llvm::raw_ostream &)> printNodeName;
+
+  /// Print the attributes used to style a node, potentially null.
+  llvm::function_ref<void (Node, llvm::raw_ostream &)> printNodeAttr;
+
+  /// Print the label for the edge set between a pair of nodes, optionally
+  /// customizing it based on the nodes themselves.
+  llvm::function_ref<
+      void (Node, Node, const SmallVectorImpl<Edge> &,
+            llvm::raw_ostream &)> printEdgeSet;
+
+public:
+  MultiGraph() = default;
+  MultiGraph(const MultiGraph &) = delete;
+  MultiGraph(MultiGraph &&) = default;
+
+  /// Insert an edge to the set of edges between two nodes.
+  void updateEdge(Node from, Node to, Edge edge) {
+    auto it = nodePairs.find(std::pair<Node, Node>{from, to});
+    if (it != nodePairs.end()) {
+      edgeSets[it->second].insert(edge);
+      return;
+    }
+    nodePairs.insert({{from, to}, edgeSets.size()});
+    EdgeSet edgeSet{};
+    edgeSet.insert(edge);
+    edgeSets.push_back(edgeSet);
+  };
+
+  /// Output the graph in DOT format to \p os.
+  void printAsGraphviz(llvm::raw_ostream &os, bool horizontal = false) {
+    os << "digraph CustomGraph {\n";
+    os << "  node [" << defaultNodeAttrs; os << "];\n";
+    if (horizontal)
+      os << "  rankdir = \"LR\";\n";
+
+    auto copyAndSort = [](const auto &set, auto &vec, auto cmp) {
+      vec.reserve(set.size());
+      llvm::copy(set, std::back_inserter(vec));
+      llvm::sort(vec, [&](auto t1, auto t2) { return cmp(t1, t2) < 0;});
+    };
+
+    // Print the nodes first in case we have custom attributes for nodes.
+    if (printNodeAttr) {
+      llvm::DenseSet<Node> nodes{};
+      for (auto &entry: nodePairs) {
+        nodes.insert(entry.first.first);
+        nodes.insert(entry.first.second);
+      }
+      std::vector<Node> nodesVec{};
+      copyAndSort(nodes, nodesVec, compareNodes);
+      for (auto node: nodes) {
+        os << "  ";
+        printNodeName(node, os);
+        printNodeAttr(node, os);
+        os << ";\n";
+      }
+    }
+    std::vector<std::pair<std::pair<Node, Node>, size_t>> nodePairsVec{};
+    copyAndSort(nodePairs, nodePairsVec,
+                [this](auto e1, auto e2) -> int {
+      auto nodePair1 = e1.first;
+      auto nodePair2 = e1.first;
+      auto cmp1 = compareNodes(nodePair1.first, nodePair2.first);
+      if (cmp1 < 0) return cmp1;
+      return compareNodes(nodePair1.second, nodePair2.second);
+    });
+    SmallVector<Edge, 5> scratchEdges;
+    for (auto &entry: nodePairsVec) {
+      auto from = entry.first.first;
+      auto to = entry.first.second;
+      auto edgeSetIndex = entry.second;
+      os << "  ";
+      printNodeName(from, os);
+      os << " -> ";
+      printNodeName(to, os);
+      if (printEdgeSet) {
+        copyAndSort(edgeSets[edgeSetIndex], scratchEdges, compareEdges);
+        os << " [label = \"";
+        printEdgeSet(from, to, scratchEdges, os);
+        os << "\"]";
+      }
+      scratchEdges.clear();
+      os << ";\n";
+    }
+    os << "}\n";
+  }
+};
+
+} // end namespace swift
+
+#endif // SWIFT_BASIC_GRAPHVIZ_H

include/swift/Basic/SupplementaryOutputPaths.h

@@ -126,6 +126,9 @@ struct SupplementaryOutputPaths {
   /// TARGET. This format is subject to arbitrary change, however.
   std::string LoadedModuleTracePath;
 
+  /// Path for storing a module import graph in Graphviz DOT format.
+  std::string ModuleImportGraphPath;
+
   /// The path to which we should output a TBD file.
   ///
   /// "TBD" stands for "text-based dylib". It's a YAML-based format that
@@ -176,7 +179,8 @@ struct SupplementaryOutputPaths {
            ModuleDocOutputPath.empty() && DependenciesFilePath.empty() &&
            ReferenceDependenciesFilePath.empty() &&
            SerializedDiagnosticsPath.empty() && LoadedModuleTracePath.empty() &&
-           TBDPath.empty() && ModuleInterfaceOutputPath.empty() &&
+           ModuleImportGraphPath.empty() && TBDPath.empty() &&
+           ModuleInterfaceOutputPath.empty() &&
            ModuleSourceInfoOutputPath.empty() && LdAddCFilePath.empty();
   }
 };

include/swift/Frontend/InputFile.h

@@ -90,6 +90,9 @@ class InputFile {
   std::string loadedModuleTracePath() const {
     return getPrimarySpecificPaths().SupplementaryOutputs.LoadedModuleTracePath;
   }
+  std::string moduleImportGraphPath() const {
+    return getPrimarySpecificPaths().SupplementaryOutputs.ModuleImportGraphPath;
+  }
   std::string serializedDiagnosticsPath() const {
     return getPrimarySpecificPaths().SupplementaryOutputs
         .SerializedDiagnosticsPath;

include/swift/Option/Options.td

@@ -337,6 +337,13 @@ def emit_loaded_module_trace_path_EQ : Joined<["-"], "emit-loaded-module-trace-p
   Flags<[FrontendOption, NoInteractiveOption, ArgumentIsPath,
          SupplementaryOutput]>,
   Alias<emit_loaded_module_trace_path>;
+def emit_module_import_graph_path : Separate<["-"],
+    "emit-module-import-graph-path">,
+  Flags<[FrontendOption, NoInteractiveOption, ArgumentIsPath,
+         SupplementaryOutput]>,
+  HelpText<"Emit a module import graph in GraphViz DOT format to <path>. The key used is Exp = @_exported, Def = default, IO = @_implementationOnly, SPI = @_spi, SCOO = shadowed by cross-import overlay, (C) = Clang module, (S) = Swift module, Ov = Swift overlay for Clang module, X-Ov = Cross-import overlay.">,
+  // GraphViz doesn't seem to have a proper way to add a legend to a graph. :(
+  MetaVarName<"<path>">;
 def emit_cross_import_remarks : Flag<["-"], "Rcross-import">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
   HelpText<"Emit a remark if a cross-import of a module is triggered.">;

lib/Basic/FileTypes.cpp

@@ -78,6 +78,7 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_ImportedModules:
   case file_types::TY_TBD:
   case file_types::TY_ModuleTrace:
+  case file_types::TY_ModuleImportGraph:
   case file_types::TY_YAMLOptRecord:
   case file_types::TY_SwiftModuleInterfaceFile:
   case file_types::TY_PrivateSwiftModuleInterfaceFile:
@@ -150,6 +151,7 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_Remapping:
   case file_types::TY_IndexData:
   case file_types::TY_ModuleTrace:
+  case file_types::TY_ModuleImportGraph:
   case file_types::TY_YAMLOptRecord:
   case file_types::TY_BitstreamOptRecord:
   case file_types::TY_SwiftModuleInterfaceFile:
@@ -202,6 +204,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
   case file_types::TY_Remapping:
   case file_types::TY_IndexData:
   case file_types::TY_ModuleTrace:
+  case file_types::TY_ModuleImportGraph:
   case file_types::TY_YAMLOptRecord:
   case file_types::TY_BitstreamOptRecord:
   case file_types::TY_JSONDependencies:

lib/Driver/Driver.cpp

@@ -2000,6 +2000,7 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       case file_types::TY_PCH:
       case file_types::TY_ImportedModules:
       case file_types::TY_ModuleTrace:
+      case file_types::TY_ModuleImportGraph:
       case file_types::TY_YAMLOptRecord:
       case file_types::TY_BitstreamOptRecord:
       case file_types::TY_SwiftModuleInterfaceFile:

lib/Driver/ToolChains.cpp

@@ -613,6 +613,7 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
   case file_types::TY_SwiftRanges:
   case file_types::TY_CompiledSource:
   case file_types::TY_ModuleTrace:
+  case file_types::TY_ModuleImportGraph:
   case file_types::TY_TBD:
   case file_types::TY_YAMLOptRecord:
   case file_types::TY_BitstreamOptRecord:
@@ -875,6 +876,7 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case file_types::TY_CompiledSource:
     case file_types::TY_Remapping:
     case file_types::TY_ModuleTrace:
+    case file_types::TY_ModuleImportGraph:
     case file_types::TY_YAMLOptRecord:
     case file_types::TY_BitstreamOptRecord:
     case file_types::TY_SwiftModuleInterfaceFile:

lib/Frontend/ArgsToFrontendOutputsConverter.cpp

@@ -297,6 +297,8 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
       options::OPT_emit_fixits_path);
   auto loadedModuleTrace = getSupplementaryFilenamesFromArguments(
       options::OPT_emit_loaded_module_trace_path);
+  auto moduleImportGraph = getSupplementaryFilenamesFromArguments(
+      options::OPT_emit_module_import_graph_path);
   auto TBD = getSupplementaryFilenamesFromArguments(options::OPT_emit_tbd_path);
   auto moduleInterfaceOutput = getSupplementaryFilenamesFromArguments(
       options::OPT_emit_module_interface_path);
@@ -310,9 +312,10 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
       options::OPT_emit_module_summary_path);
   if (!objCHeaderOutput || !moduleOutput || !moduleDocOutput ||
       !dependenciesFile || !referenceDependenciesFile ||
-      !serializedDiagnostics || !fixItsOutput || !loadedModuleTrace || !TBD ||
-      !moduleInterfaceOutput || !privateModuleInterfaceOutput ||
-      !moduleSourceInfoOutput || !ldAddCFileOutput || !moduleSummaryOutput) {
+      !serializedDiagnostics || !fixItsOutput || !loadedModuleTrace ||
+      !moduleImportGraph || !TBD || !moduleInterfaceOutput ||
+      !privateModuleInterfaceOutput || !moduleSourceInfoOutput ||
+      !ldAddCFileOutput || !moduleSummaryOutput) {
     return None;
   }
   std::vector<SupplementaryOutputPaths> result;
@@ -331,6 +334,7 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
     sop.SerializedDiagnosticsPath = (*serializedDiagnostics)[i];
     sop.FixItsOutputPath = (*fixItsOutput)[i];
     sop.LoadedModuleTracePath = (*loadedModuleTrace)[i];
+    sop.ModuleImportGraphPath = (*moduleImportGraph)[i];
     sop.TBDPath = (*TBD)[i];
     sop.ModuleInterfaceOutputPath = (*moduleInterfaceOutput)[i];
     sop.PrivateModuleInterfaceOutputPath = (*privateModuleInterfaceOutput)[i];
@@ -412,6 +416,12 @@ SupplementaryOutputPathsComputer::computeOutputPathsForOneInput(
       file_types::TY_ModuleTrace, "",
       defaultSupplementaryOutputPathExcludingExtension);
 
+  auto moduleImportGraphPath = determineSupplementaryOutputFilename(
+      OPT_emit_module_import_graph_path,
+      pathsFromArguments.ModuleImportGraphPath,
+      file_types::TY_ModuleImportGraph, "",
+      defaultSupplementaryOutputPathExcludingExtension);
+
   auto tbdPath = determineSupplementaryOutputFilename(
       OPT_emit_tbd, pathsFromArguments.TBDPath, file_types::TY_TBD, "",
       defaultSupplementaryOutputPathExcludingExtension);
@@ -458,6 +468,7 @@ SupplementaryOutputPathsComputer::computeOutputPathsForOneInput(
   sop.SerializedDiagnosticsPath = serializedDiagnosticsPath;
   sop.FixItsOutputPath = fixItsOutputPath;
   sop.LoadedModuleTracePath = loadedModuleTracePath;
+  sop.ModuleImportGraphPath = moduleImportGraphPath;
   sop.TBDPath = tbdPath;
   sop.ModuleInterfaceOutputPath = ModuleInterfaceOutputPath;
   sop.PrivateModuleInterfaceOutputPath = PrivateModuleInterfaceOutputPath;
@@ -567,6 +578,7 @@ SupplementaryOutputPathsComputer::readSupplementaryOutputFileMap() const {
         options::OPT_emit_swift_ranges_path,
         options::OPT_serialize_diagnostics_path,
         options::OPT_emit_loaded_module_trace_path,
+        options::OPT_emit_module_import_graph_path,
         options::OPT_emit_module_interface_path,
         options::OPT_emit_private_module_interface_path,
         options::OPT_emit_module_source_info_path,