Skip to content

[ModuleTrace] Emit import graph to ease debugging import issues. #33980

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions include/swift/AST/Module.h
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,15 @@ class ModuleDecl : public DeclContext, public TypeDecl {
/// this flag is specified.
ShadowedBySeparateOverlay = 1 << 4
};

static constexpr std::array<ImportFilterKind, 5> AllImportFilterKinds {
ImportFilterKind::Public,
ImportFilterKind::Private,
ImportFilterKind::ImplementationOnly,
ImportFilterKind::SPIAccessControl,
ImportFilterKind::ShadowedBySeparateOverlay,
};

/// \sa getImportedModules
using ImportFilter = OptionSet<ImportFilterKind>;

Expand Down
2 changes: 2 additions & 0 deletions include/swift/Basic/FileTypes.def
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ TYPE("tbd", TBD, "tbd", "")
// Swift section of the internal wiki.
TYPE("module-trace", ModuleTrace, "trace.json", "")

TYPE("module-import-graph", ModuleImportGraph, "import-graph.dot","")

// Complete dependency information for the given Swift files as JSON.
TYPE("json-dependencies", JSONDependencies, "dependencies.json", "")

Expand Down
152 changes: 152 additions & 0 deletions include/swift/Basic/GraphViz.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
//===--- GraphViz.h - Utilities for outputting graphs -----------*- C++ -*-===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Defines data structures for creating transient graphs in order to print
/// them to GraphViz DOT format.
///
//===----------------------------------------------------------------------===//

#ifndef SWIFT_BASIC_GRAPHVIZ_H
#define SWIFT_BASIC_GRAPHVIZ_H

#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/raw_ostream.h"

#include <iterator>
#include <string>
#include <utility>
#include <vector>

namespace swift {
// MARK: - MultiGraph

/// A multigraph data structure, with a set of edges connecting pairs of nodes.
///
/// The fields are left public so that you can potentially customize printing
/// directly, instead of relying on the existing customization points.
///
/// \invariant {
/// \code forall n1 n2. nodePairs[{n1, n2}] < edgeSets.size() \endcode
/// }
template <typename Node, typename Edge, typename EdgeSet>
struct MultiGraph {
/// List of nodes stored in the graph.
llvm::DenseMap<std::pair<Node, Node>, size_t> nodePairs;

/// List of edges stored in the graph.
std::vector<EdgeSet> edgeSets;

/// Three way comparison function for nodes used for deterministic output.
llvm::function_ref<int (Node, Node)> compareNodes;

/// Three way comparison function for edges used for deterministic output.
llvm::function_ref<int (Edge, Edge)> compareEdges;

/// Default attributes used for styling nodes.
std::string defaultNodeAttrs =
"shape = \"box\", style = \"rounded\", penwidth = \"2\"";

/// Print the name of a node.
llvm::function_ref<void (Node, llvm::raw_ostream &)> printNodeName;

/// Print the attributes used to style a node, potentially null.
llvm::function_ref<void (Node, llvm::raw_ostream &)> printNodeAttr;

/// Print the label for the edge set between a pair of nodes, optionally
/// customizing it based on the nodes themselves.
llvm::function_ref<
void (Node, Node, const SmallVectorImpl<Edge> &,
llvm::raw_ostream &)> printEdgeSet;

public:
MultiGraph() = default;
MultiGraph(const MultiGraph &) = delete;
MultiGraph(MultiGraph &&) = default;

/// Insert an edge to the set of edges between two nodes.
void updateEdge(Node from, Node to, Edge edge) {
auto it = nodePairs.find(std::pair<Node, Node>{from, to});
if (it != nodePairs.end()) {
edgeSets[it->second].insert(edge);
return;
}
nodePairs.insert({{from, to}, edgeSets.size()});
EdgeSet edgeSet{};
edgeSet.insert(edge);
edgeSets.push_back(edgeSet);
};

/// Output the graph in DOT format to \p os.
void printAsGraphviz(llvm::raw_ostream &os, bool horizontal = false) {
os << "digraph CustomGraph {\n";
os << " node [" << defaultNodeAttrs; os << "];\n";
if (horizontal)
os << " rankdir = \"LR\";\n";

auto copyAndSort = [](const auto &set, auto &vec, auto cmp) {
vec.reserve(set.size());
llvm::copy(set, std::back_inserter(vec));
llvm::sort(vec, [&](auto t1, auto t2) { return cmp(t1, t2) < 0;});
};

// Print the nodes first in case we have custom attributes for nodes.
if (printNodeAttr) {
llvm::DenseSet<Node> nodes{};
for (auto &entry: nodePairs) {
nodes.insert(entry.first.first);
nodes.insert(entry.first.second);
}
std::vector<Node> nodesVec{};
copyAndSort(nodes, nodesVec, compareNodes);
for (auto node: nodes) {
os << " ";
printNodeName(node, os);
printNodeAttr(node, os);
os << ";\n";
}
}
std::vector<std::pair<std::pair<Node, Node>, size_t>> nodePairsVec{};
copyAndSort(nodePairs, nodePairsVec,
[this](auto e1, auto e2) -> int {
auto nodePair1 = e1.first;
auto nodePair2 = e1.first;
auto cmp1 = compareNodes(nodePair1.first, nodePair2.first);
if (cmp1 < 0) return cmp1;
return compareNodes(nodePair1.second, nodePair2.second);
});
SmallVector<Edge, 5> scratchEdges;
for (auto &entry: nodePairsVec) {
auto from = entry.first.first;
auto to = entry.first.second;
auto edgeSetIndex = entry.second;
os << " ";
printNodeName(from, os);
os << " -> ";
printNodeName(to, os);
if (printEdgeSet) {
copyAndSort(edgeSets[edgeSetIndex], scratchEdges, compareEdges);
os << " [label = \"";
printEdgeSet(from, to, scratchEdges, os);
os << "\"]";
}
scratchEdges.clear();
os << ";\n";
}
os << "}\n";
}
};

} // end namespace swift

#endif // SWIFT_BASIC_GRAPHVIZ_H
6 changes: 5 additions & 1 deletion include/swift/Basic/SupplementaryOutputPaths.h
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,9 @@ struct SupplementaryOutputPaths {
/// TARGET. This format is subject to arbitrary change, however.
std::string LoadedModuleTracePath;

/// Path for storing a module import graph in Graphviz DOT format.
std::string ModuleImportGraphPath;

/// The path to which we should output a TBD file.
///
/// "TBD" stands for "text-based dylib". It's a YAML-based format that
Expand Down Expand Up @@ -176,7 +179,8 @@ struct SupplementaryOutputPaths {
ModuleDocOutputPath.empty() && DependenciesFilePath.empty() &&
ReferenceDependenciesFilePath.empty() &&
SerializedDiagnosticsPath.empty() && LoadedModuleTracePath.empty() &&
TBDPath.empty() && ModuleInterfaceOutputPath.empty() &&
ModuleImportGraphPath.empty() && TBDPath.empty() &&
ModuleInterfaceOutputPath.empty() &&
ModuleSourceInfoOutputPath.empty() && LdAddCFilePath.empty();
}
};
Expand Down
3 changes: 3 additions & 0 deletions include/swift/Frontend/InputFile.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ class InputFile {
std::string loadedModuleTracePath() const {
return getPrimarySpecificPaths().SupplementaryOutputs.LoadedModuleTracePath;
}
std::string moduleImportGraphPath() const {
return getPrimarySpecificPaths().SupplementaryOutputs.ModuleImportGraphPath;
}
std::string serializedDiagnosticsPath() const {
return getPrimarySpecificPaths().SupplementaryOutputs
.SerializedDiagnosticsPath;
Expand Down
7 changes: 7 additions & 0 deletions include/swift/Option/Options.td
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,13 @@ def emit_loaded_module_trace_path_EQ : Joined<["-"], "emit-loaded-module-trace-p
Flags<[FrontendOption, NoInteractiveOption, ArgumentIsPath,
SupplementaryOutput]>,
Alias<emit_loaded_module_trace_path>;
def emit_module_import_graph_path : Separate<["-"],
"emit-module-import-graph-path">,
Flags<[FrontendOption, NoInteractiveOption, ArgumentIsPath,
SupplementaryOutput]>,
HelpText<"Emit a module import graph in GraphViz DOT format to <path>. The key used is Exp = @_exported, Def = default, IO = @_implementationOnly, SPI = @_spi, SCOO = shadowed by cross-import overlay, (C) = Clang module, (S) = Swift module, Ov = Swift overlay for Clang module, X-Ov = Cross-import overlay.">,
// GraphViz doesn't seem to have a proper way to add a legend to a graph. :(
MetaVarName<"<path>">;
def emit_cross_import_remarks : Flag<["-"], "Rcross-import">,
Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
HelpText<"Emit a remark if a cross-import of a module is triggered.">;
Expand Down
5 changes: 2 additions & 3 deletions lib/AST/Module.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1191,12 +1191,11 @@ SourceFile::getImportedModules(SmallVectorImpl<ModuleDecl::ImportedModule> &modu
requiredFilter |= ModuleDecl::ImportFilterKind::ImplementationOnly;
else if (desc.importOptions.contains(ImportFlags::SPIAccessControl))
requiredFilter |= ModuleDecl::ImportFilterKind::SPIAccessControl;
else if (!separatelyImportedOverlays.lookup(desc.module.importedModule).empty())
requiredFilter |= ModuleDecl::ImportFilterKind::ShadowedBySeparateOverlay;
else
requiredFilter |= ModuleDecl::ImportFilterKind::Private;

if (!separatelyImportedOverlays.lookup(desc.module.importedModule).empty())
requiredFilter |= ModuleDecl::ImportFilterKind::ShadowedBySeparateOverlay;

Comment on lines +1194 to -1199
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This "fix" is incorrect. The ShadowedBySeparateOverlay filter is essentially a different axis altogether.

             | @_exported | private | @_implementationOnly | @_spi
-------------+------------+---------+----------------------+------
Not shadowed |     EN     |   PN    |        IN            |  SN
Shadowed     |     ES     |   PS    |        IS            |  SS

Ideally, we'd be passing in two OptionSets, not one IMO. If you pass in newFilter = {{@_exported}, {not shadowed}}, you get EN. If you pass in newFilter = {{@_exported | private}, {shadowed}}, you'd get ES and PS.

Today, if you pass in filter = {@_exported}, you get EN whereas if you pass in filter = {shadowed}, you get nothing. That's unnecessarily confusing.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, @_spi is a different axis, so the table in ^ is wrong.

if (filter.contains(requiredFilter))
modules.push_back(desc.module);
}
Expand Down
3 changes: 3 additions & 0 deletions lib/Basic/FileTypes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ bool file_types::isTextual(ID Id) {
case file_types::TY_ImportedModules:
case file_types::TY_TBD:
case file_types::TY_ModuleTrace:
case file_types::TY_ModuleImportGraph:
case file_types::TY_YAMLOptRecord:
case file_types::TY_SwiftModuleInterfaceFile:
case file_types::TY_PrivateSwiftModuleInterfaceFile:
Expand Down Expand Up @@ -150,6 +151,7 @@ bool file_types::isAfterLLVM(ID Id) {
case file_types::TY_Remapping:
case file_types::TY_IndexData:
case file_types::TY_ModuleTrace:
case file_types::TY_ModuleImportGraph:
case file_types::TY_YAMLOptRecord:
case file_types::TY_BitstreamOptRecord:
case file_types::TY_SwiftModuleInterfaceFile:
Expand Down Expand Up @@ -202,6 +204,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
case file_types::TY_Remapping:
case file_types::TY_IndexData:
case file_types::TY_ModuleTrace:
case file_types::TY_ModuleImportGraph:
case file_types::TY_YAMLOptRecord:
case file_types::TY_BitstreamOptRecord:
case file_types::TY_JSONDependencies:
Expand Down
1 change: 1 addition & 0 deletions lib/Driver/Driver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2000,6 +2000,7 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
case file_types::TY_PCH:
case file_types::TY_ImportedModules:
case file_types::TY_ModuleTrace:
case file_types::TY_ModuleImportGraph:
case file_types::TY_YAMLOptRecord:
case file_types::TY_BitstreamOptRecord:
case file_types::TY_SwiftModuleInterfaceFile:
Expand Down
2 changes: 2 additions & 0 deletions lib/Driver/ToolChains.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -613,6 +613,7 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
case file_types::TY_SwiftRanges:
case file_types::TY_CompiledSource:
case file_types::TY_ModuleTrace:
case file_types::TY_ModuleImportGraph:
case file_types::TY_TBD:
case file_types::TY_YAMLOptRecord:
case file_types::TY_BitstreamOptRecord:
Expand Down Expand Up @@ -875,6 +876,7 @@ ToolChain::constructInvocation(const BackendJobAction &job,
case file_types::TY_CompiledSource:
case file_types::TY_Remapping:
case file_types::TY_ModuleTrace:
case file_types::TY_ModuleImportGraph:
case file_types::TY_YAMLOptRecord:
case file_types::TY_BitstreamOptRecord:
case file_types::TY_SwiftModuleInterfaceFile:
Expand Down
18 changes: 15 additions & 3 deletions lib/Frontend/ArgsToFrontendOutputsConverter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,8 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
options::OPT_emit_fixits_path);
auto loadedModuleTrace = getSupplementaryFilenamesFromArguments(
options::OPT_emit_loaded_module_trace_path);
auto moduleImportGraph = getSupplementaryFilenamesFromArguments(
options::OPT_emit_module_import_graph_path);
auto TBD = getSupplementaryFilenamesFromArguments(options::OPT_emit_tbd_path);
auto moduleInterfaceOutput = getSupplementaryFilenamesFromArguments(
options::OPT_emit_module_interface_path);
Expand All @@ -310,9 +312,10 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
options::OPT_emit_module_summary_path);
if (!objCHeaderOutput || !moduleOutput || !moduleDocOutput ||
!dependenciesFile || !referenceDependenciesFile ||
!serializedDiagnostics || !fixItsOutput || !loadedModuleTrace || !TBD ||
!moduleInterfaceOutput || !privateModuleInterfaceOutput ||
!moduleSourceInfoOutput || !ldAddCFileOutput || !moduleSummaryOutput) {
!serializedDiagnostics || !fixItsOutput || !loadedModuleTrace ||
!moduleImportGraph || !TBD || !moduleInterfaceOutput ||
!privateModuleInterfaceOutput || !moduleSourceInfoOutput ||
!ldAddCFileOutput || !moduleSummaryOutput) {
return None;
}
std::vector<SupplementaryOutputPaths> result;
Expand All @@ -331,6 +334,7 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
sop.SerializedDiagnosticsPath = (*serializedDiagnostics)[i];
sop.FixItsOutputPath = (*fixItsOutput)[i];
sop.LoadedModuleTracePath = (*loadedModuleTrace)[i];
sop.ModuleImportGraphPath = (*moduleImportGraph)[i];
sop.TBDPath = (*TBD)[i];
sop.ModuleInterfaceOutputPath = (*moduleInterfaceOutput)[i];
sop.PrivateModuleInterfaceOutputPath = (*privateModuleInterfaceOutput)[i];
Expand Down Expand Up @@ -412,6 +416,12 @@ SupplementaryOutputPathsComputer::computeOutputPathsForOneInput(
file_types::TY_ModuleTrace, "",
defaultSupplementaryOutputPathExcludingExtension);

auto moduleImportGraphPath = determineSupplementaryOutputFilename(
OPT_emit_module_import_graph_path,
pathsFromArguments.ModuleImportGraphPath,
file_types::TY_ModuleImportGraph, "",
defaultSupplementaryOutputPathExcludingExtension);

auto tbdPath = determineSupplementaryOutputFilename(
OPT_emit_tbd, pathsFromArguments.TBDPath, file_types::TY_TBD, "",
defaultSupplementaryOutputPathExcludingExtension);
Expand Down Expand Up @@ -458,6 +468,7 @@ SupplementaryOutputPathsComputer::computeOutputPathsForOneInput(
sop.SerializedDiagnosticsPath = serializedDiagnosticsPath;
sop.FixItsOutputPath = fixItsOutputPath;
sop.LoadedModuleTracePath = loadedModuleTracePath;
sop.ModuleImportGraphPath = moduleImportGraphPath;
sop.TBDPath = tbdPath;
sop.ModuleInterfaceOutputPath = ModuleInterfaceOutputPath;
sop.PrivateModuleInterfaceOutputPath = PrivateModuleInterfaceOutputPath;
Expand Down Expand Up @@ -567,6 +578,7 @@ SupplementaryOutputPathsComputer::readSupplementaryOutputFileMap() const {
options::OPT_emit_swift_ranges_path,
options::OPT_serialize_diagnostics_path,
options::OPT_emit_loaded_module_trace_path,
options::OPT_emit_module_import_graph_path,
options::OPT_emit_module_interface_path,
options::OPT_emit_private_module_interface_path,
options::OPT_emit_module_source_info_path,
Expand Down
Loading