Merge pull request swiftlang#29126 from davidungar/fine-grained-fixes-rb2-w-dbg

[Incremental] Dependency fixes in preparation for fine-grained dependencies

Author: David Ungar

include/swift/AST/FineGrainedDependencies.h

@@ -325,7 +325,7 @@ class BiIndexedTwoStageMap {
 /// Write out the .swiftdeps file for a frontend compilation of a primary file.
 bool emitReferenceDependencies(DiagnosticEngine &diags, SourceFile *SF,
                                const DependencyTracker &depTracker,
-                               StringRef outputPath);
+                               StringRef outputPath, bool alsoEmitDotFile);
 //==============================================================================
 // MARK: Enums
 //==============================================================================
@@ -438,8 +438,8 @@ class DependencyKey {
         name() {}
 
   /// For constructing a key in the frontend.
-  DependencyKey(NodeKind kind, DeclAspect aspect, std::string context,
-                std::string name)
+  DependencyKey(NodeKind kind, DeclAspect aspect, const std::string &context,
+                const std::string &name)
       : kind(kind), aspect(aspect), context(context), name(name) {
     assert(verify());
   }
@@ -449,7 +449,7 @@ class DependencyKey {
   StringRef getContext() const { return context; }
   StringRef getName() const { return name; }
 
-  StringRef getSwiftDepsFromSourceFileProvide() const {
+  StringRef getSwiftDepsFromASourceFileProvideNodeKey() const {
     assert(getKind() == NodeKind::sourceFileProvide &&
            "Receiver must be sourceFileProvide.");
     return getName();
@@ -494,11 +494,19 @@ class DependencyKey {
   static std::string computeNameForProvidedEntity(Entity);
 
   /// Given some type of depended-upon entity create the key.
-  template <NodeKind kind, typename Entity>
-  static DependencyKey createDependedUponKey(const Entity &);
+  static DependencyKey createDependedUponKey(StringRef mangledHolderName,
+                                             StringRef memberBaseName);
+
+  template <NodeKind kind>
+  static DependencyKey createDependedUponKey(StringRef);
+
+  static DependencyKey
+  createTransitiveKeyForWholeSourceFile(StringRef swiftDeps);
 
   std::string humanReadableName() const;
 
+  StringRef aspectName() const { return DeclAspectNames[size_t(aspect)]; }
+
   void dump(llvm::raw_ostream &os) const { os << asString() << "\n"; }
   SWIFT_DEBUG_DUMP { dump(llvm::errs()); }
 
@@ -535,6 +543,13 @@ struct std::hash<typename swift::fine_grained_dependencies::DeclAspect> {
   }
 };
 
+namespace swift {
+namespace fine_grained_dependencies {
+using ContextNameFingerprint =
+    std::tuple<std::string, std::string, Optional<std::string>>;
+}
+} // namespace swift
+
 //==============================================================================
 // MARK: DepGraphNode
 //==============================================================================
@@ -684,6 +699,9 @@ class SourceFileDepGraphNode : public DepGraphNode {
   }
 
   /// Record the sequence number, \p n, of another use.
+  /// The relationship between an interface and its implementation is NOT
+  /// included here. See \c
+  /// SourceFileDepGraph::findExistingNodePairOrCreateAndAddIfNew.
   void addDefIDependUpon(size_t n) {
     if (n != getSequenceNumber())
       defsIDependUpon.insert(n);
@@ -696,8 +714,29 @@ class SourceFileDepGraphNode : public DepGraphNode {
   bool verify() const {
     DepGraphNode::verify();
     assert(getIsProvides() || isDepends());
+    assert(verifySequenceNumber());
     return true;
   }
+
+  bool verifySequenceNumber() const {
+    const auto &k = getKey();
+    if (k.getKind() != NodeKind::sourceFileProvide)
+      return true;
+    switch (k.getAspect()) {
+    case DeclAspect::interface:
+      assert(getSequenceNumber() == sourceFileProvidesInterfaceSequenceNumber);
+      return true;
+    case DeclAspect::implementation:
+      assert(getSequenceNumber() ==
+             sourceFileProvidesImplementationSequenceNumber);
+      return true;
+    default:
+      llvm_unreachable("neither interface nor implementation");
+    }
+  }
+  static constexpr const size_t sourceFileProvidesInterfaceSequenceNumber = 0;
+  static constexpr const size_t sourceFileProvidesImplementationSequenceNumber =
+      1;
 };
 
 //==============================================================================
@@ -727,6 +766,25 @@ class SourceFileDepGraph {
   SourceFileDepGraph(const SourceFileDepGraph &g) = delete;
   SourceFileDepGraph(SourceFileDepGraph &&g) = default;
 
+  /// Simulate loading for unit testing:
+  /// \param swiftDepsFileName The name of the swiftdeps file of the phony job
+  /// \param includePrivateDeps Whether the graph includes intra-file arcs
+  /// \param hadCompilationError Simulate a compilation error
+  /// \param interfaceHash The interface hash of the simulated graph
+  /// \param simpleNamesByRDK A map of vectors of names keyed by reference
+  /// dependency key \param compoundNamesByRDK A map of (mangledHolder,
+  /// baseName) pairs keyed by reference dependency key. For single-name
+  /// dependencies, an initial underscore indicates that the name does not
+  /// cascade. For compound names, it is the first name, the holder which
+  /// indicates non-cascading. For member names, an initial underscore indicates
+  /// file-privacy.
+  static SourceFileDepGraph
+  simulateLoad(std::string swiftDepsFileName, const bool includePrivateDeps,
+               const bool hadCompilationError, std::string interfaceHash,
+               llvm::StringMap<std::vector<std::string>> simpleNamesByRDK,
+               llvm::StringMap<std::vector<std::pair<std::string, std::string>>>
+                   compoundNamesByRDK);
+
   /// Nodes are owned by the graph.
   ~SourceFileDepGraph() {
     forEachNode([&](SourceFileDepGraphNode *n) { delete n; });
@@ -746,7 +804,7 @@ class SourceFileDepGraph {
   InterfaceAndImplementationPair<SourceFileDepGraphNode>
   getSourceFileNodePair() const;
 
-  StringRef getSwiftDepsFromSourceFileProvide() const;
+  StringRef getSwiftDepsOfJobThatProducedThisGraph() const;
 
   std::string getGraphID() const {
     return getSourceFileNodePair().getInterface()->getKey().humanReadableName();
@@ -770,12 +828,13 @@ class SourceFileDepGraph {
   /// The frontend creates a pair of nodes for every tracked Decl and the source
   /// file itself.
   InterfaceAndImplementationPair<SourceFileDepGraphNode>
-  findExistingNodePairOrCreateAndAddIfNew(NodeKind k, StringRef context,
-                                          StringRef name,
-                                          Optional<std::string> fingerprint);
+  findExistingNodePairOrCreateAndAddIfNew(
+      NodeKind k, const ContextNameFingerprint &contextNameFingerprint);
 
-  SourceFileDepGraphNode *findExistingNodeOrCreateIfNew(
-      DependencyKey key, Optional<std::string> fingerprint, bool isProvides);
+  SourceFileDepGraphNode *
+  findExistingNodeOrCreateIfNew(DependencyKey key,
+                                const Optional<std::string> &fingerprint,
+                                bool isProvides);
 
   /// \p Use is the Node that must be rebuilt when \p def changes.
   /// Record that fact in the graph.
@@ -800,14 +859,14 @@ class SourceFileDepGraph {
   /// Ensure that when read, the graph is the same as what was written.
   bool verifyReadsWhatIsWritten(StringRef path) const;
 
+  bool verifySequenceNumber() const;
+
 private:
   void addNode(SourceFileDepGraphNode *n) {
     n->setSequenceNumber(allNodes.size());
-    assert(allNodes.size() < 2 ==
-               (n->getKey().getKind() == NodeKind::sourceFileProvide) &&
-           "First two and only first two nodes should be sourceFileProvide "
-           "nodes.");
     allNodes.push_back(n);
+    assert(n->verifySequenceNumber() &&
+           "Certain nodes must be in certain places");
   }
 };
 
@@ -892,7 +951,7 @@ template <typename GraphT> class DotFileEmitter {
   }
   void emitArcs() {
     g.forEachArc([&](const NodeT *def, const NodeT *use) {
-      if (includeGraphArc(use, def))
+      if (includeGraphArc(def, use))
         emitGraphArc(def, use);
     });
   }

include/swift/Basic/LangOptions.h

@@ -285,10 +285,14 @@ namespace swift {
     /// Whether to verify the parsed syntax tree and emit related diagnostics.
     bool VerifySyntaxTree = false;
 
-    /// Scaffolding to permit experimentation with finer-grained dependencies
-    /// and faster rebuilds.
+    /// Emit the newer, finer-grained swiftdeps file. Eventually will support
+    /// faster rebuilds.
     bool EnableFineGrainedDependencies = false;
-    
+
+    /// When using fine-grained dependencies, emit dot files for every swiftdeps
+    /// file.
+    bool EmitFineGrainedDependencySourcefileDotFiles = false;
+
     /// To mimic existing system, set to false.
     /// To experiment with including file-private and private dependency info,
     /// set to true.

include/swift/Driver/Action.h

@@ -168,6 +168,23 @@ class CompileJobAction : public JobAction {
   static bool classof(const Action *A) {
     return A->getKind() == Action::Kind::CompileJob;
   }
+
+  /// Return a _single_ TY_Swift InputAction, if one exists;
+  /// if 0 or >1 such inputs exist, return nullptr.
+  const InputAction *findSingleSwiftInput() const {
+    auto Inputs = getInputs();
+    auto isSwiftInput = [](const Action *A) -> const InputAction* {
+      if (auto const *S = dyn_cast<InputAction>(A))
+        return S->getType() == file_types::TY_Swift ? S : nullptr;
+      return nullptr;
+    };
+    const auto loc1 = std::find_if(Inputs.begin(), Inputs.end(), isSwiftInput);
+    if (loc1 == Inputs.end())
+      return nullptr; // none found
+    // Ensure uniqueness
+    const auto loc2 = std::find_if(loc1 + 1, Inputs.end(), isSwiftInput);
+    return loc2 == Inputs.end() ? dyn_cast<InputAction>(*loc1) : nullptr;
+  }
 };
 
 class InterpretJobAction : public JobAction {

include/swift/Driver/Compilation.h

@@ -521,6 +521,18 @@ class Compilation {
   /// How many .swift input files?
   unsigned countSwiftInputs() const;
 
+  /// Unfortunately the success or failure of a Swift compilation is currently
+  /// sensitive to the order in which files are processed, at least in terms of
+  /// the order of processing extensions (and likely other ways we haven't
+  /// discovered yet). So long as this is true, we need to make sure any batch
+  /// job we build names its inputs in an order that's a subsequence of the
+  /// sequence of inputs the driver was initially invoked with.
+  ///
+  /// Also use to write out information in a consistent order.
+  void sortJobsToMatchCompilationInputs(
+      ArrayRef<const Job *> unsortedJobs,
+      SmallVectorImpl<const Job *> &sortedJobs) const;
+
 private:
   /// Perform all jobs.
   ///