Merge pull request #31568 from CodaFi/this-is-a-private-party

Initial Infrastructure for "Private Intransitive" Dependencies

Author: CodaFi

include/swift/AST/Evaluator.h

@@ -245,7 +245,8 @@ class Evaluator {
   /// diagnostics through the given diagnostics engine.
   Evaluator(DiagnosticEngine &diags,
             bool debugDumpCycles,
-            bool buildDependencyGraph);
+            bool buildDependencyGraph,
+            bool enableExperimentalPrivateDeps);
 
   /// Emit GraphViz output visualizing the request graph.
   void emitRequestEvaluatorGraphViz(llvm::StringRef graphVizPath);

include/swift/AST/EvaluatorDependencies.h

@@ -113,7 +113,22 @@ struct DependencyCollector {
   llvm::SmallVector<evaluator::DependencySource, 8> dependencySources;
 
 public:
-  DependencyCollector() = default;
+  enum class Mode {
+    // Enables the current "status quo" behavior of the dependency collector.
+    //
+    // By default, the dependency collector moves to register dependencies in
+    // the referenced name trackers at the top of the active dependency stack.
+    StatusQuo,
+    // Enables an experimental mode to only register private dependencies.
+    //
+    // This mode restricts the dependency collector to ignore changes of
+    // scope. This has practical effect of charging all unqualified lookups to
+    // the primary file being acted upon instead of to the destination file.
+    ExperimentalPrivateDependencies,
+  };
+  Mode mode;
+
+  explicit DependencyCollector(Mode mode) : mode{mode} {};
 
 public:
   /// Registers a named reference from the current dependency scope to a member
@@ -206,19 +221,43 @@ struct DependencyCollector {
   };
 
 private:
+  /// Returns the first dependency source registered with the tracker, or
+  /// \c nullptr if no dependency sources have been registered.
+  SourceFile *getFirstDependencySourceOrNull() const {
+    if (dependencySources.empty())
+      return nullptr;
+    return dependencySources.front().getPointer();
+  }
+
   /// If there is an active dependency source, returns its
   /// \c ReferencedNameTracker. Else, returns \c nullptr.
   ReferencedNameTracker *getActiveDependencyTracker() const {
-    if (auto *source = getActiveDependencySourceOrNull())
-      return source->getRequestBasedReferencedNameTracker();
-    return nullptr;
+    SourceFile *source = nullptr;
+    switch (mode) {
+    case Mode::StatusQuo:
+      source = getActiveDependencySourceOrNull();
+      break;
+    case Mode::ExperimentalPrivateDependencies:
+      source = getFirstDependencySourceOrNull();
+      break;
+    }
+    
+    if (!source)
+      return nullptr;
+
+    return source->getRequestBasedReferencedNameTracker();
   }
 
   /// Returns \c true if the scope of the current active source cascades.
   ///
   /// If there is no active scope, the result always cascades.
   bool isActiveSourceCascading() const {
-    return getActiveSourceScope() == evaluator::DependencyScope::Cascading;
+    switch (mode) {
+    case Mode::StatusQuo:
+      return getActiveSourceScope() == evaluator::DependencyScope::Cascading;
+    case Mode::ExperimentalPrivateDependencies:
+      return false;
+    }
   }
 };
 } // end namespace evaluator

include/swift/Basic/LangOptions.h

@@ -357,6 +357,10 @@ namespace swift {
     /// conformances.
     bool EnableExperimentalAdditiveArithmeticDerivedConformances = false;
 
+    /// Whether to enable a more aggressive mode of incremental dependency
+    /// gathering that never captures cascading edges.
+    bool EnableExperientalPrivateIntransitiveDependencies = false;
+
     /// Enable verification when every SubstitutionMap is constructed.
     bool VerifyAllSubstitutionMaps = false;
 

include/swift/Option/Options.td

@@ -511,6 +511,12 @@ def enable_experimental_concise_pound_file : Flag<["-"],
   Flags<[FrontendOption]>,
   HelpText<"Enable experimental concise '#file' identifier">;
 
+def experimental_private_intransitive_dependencies : Flag<["-"],
+    "experimental-private-intransitive-dependencies">,
+  Flags<[FrontendOption, HelpHidden]>,
+  HelpText<"Enable experimental dependency tracking that never cascades">;
+
+
 // Diagnostic control options
 def suppress_warnings : Flag<["-"], "suppress-warnings">,
   Flags<[FrontendOption]>,

lib/AST/ASTContext.cpp

@@ -552,7 +552,8 @@ ASTContext::ASTContext(LangOptions &langOpts, TypeCheckerOptions &typeckOpts,
     SearchPathOpts(SearchPathOpts), SourceMgr(SourceMgr), Diags(Diags),
     evaluator(Diags,
               langOpts.DebugDumpCycles,
-              langOpts.BuildRequestDependencyGraph),
+              langOpts.BuildRequestDependencyGraph,
+              langOpts.EnableExperientalPrivateIntransitiveDependencies),
     TheBuiltinModule(createBuiltinModule(*this)),
     StdlibModuleName(getIdentifier(STDLIB_NAME)),
     SwiftShimsModuleName(getIdentifier(SWIFT_SHIMS_NAME)),

lib/AST/Evaluator.cpp

@@ -60,12 +60,22 @@ void Evaluator::registerRequestFunctions(
   requestFunctionsByZone.push_back({zoneID, functions});
 }
 
-Evaluator::Evaluator(DiagnosticEngine &diags,
-                     bool debugDumpCycles,
-                     bool buildDependencyGraph)
-  : diags(diags),
-    debugDumpCycles(debugDumpCycles),
-    buildDependencyGraph(buildDependencyGraph) { }
+static evaluator::DependencyCollector::Mode
+computeDependencyModeFromFlags(bool enableExperimentalPrivateDeps) {
+  using Mode = evaluator::DependencyCollector::Mode;
+  if (enableExperimentalPrivateDeps) {
+    return Mode::ExperimentalPrivateDependencies;
+  }
+  return Mode::StatusQuo;
+}
+
+Evaluator::Evaluator(DiagnosticEngine &diags, bool debugDumpCycles,
+                     bool buildDependencyGraph,
+                     bool enableExperimentalPrivateDeps)
+    : diags(diags), debugDumpCycles(debugDumpCycles),
+      buildDependencyGraph(buildDependencyGraph),
+      collector{computeDependencyModeFromFlags(enableExperimentalPrivateDeps)} {
+}
 
 void Evaluator::emitRequestEvaluatorGraphViz(llvm::StringRef graphVizPath) {
   std::error_code error;

lib/Driver/ToolChains.cpp

@@ -250,7 +250,9 @@ void ToolChain::addCommonFrontendArgs(const OutputInfo &OI,
                        options::OPT_enable_experimental_concise_pound_file);
   inputArgs.AddLastArg(arguments,
                        options::OPT_verify_incremental_dependencies);
-  
+  inputArgs.AddLastArg(arguments,
+                       options::OPT_experimental_private_intransitive_dependencies);
+
   // Pass on any build config options
   inputArgs.AddAllArgs(arguments, options::OPT_D);
 

lib/Frontend/CompilerInvocation.cpp

@@ -434,6 +434,9 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
   if (Args.hasArg(OPT_enable_experimental_additive_arithmetic_derivation))
     Opts.EnableExperimentalAdditiveArithmeticDerivedConformances = true;
 
+  if (Args.hasArg(OPT_experimental_private_intransitive_dependencies))
+    Opts.EnableExperientalPrivateIntransitiveDependencies = true;
+
   Opts.EnableExperimentalForwardModeDifferentiation |=
       Args.hasArg(OPT_enable_experimental_forward_mode_differentiation);
 

lib/FrontendTool/FrontendTool.cpp

@@ -791,10 +791,9 @@ static bool compileLLVMIR(CompilerInstance &Instance) {
                                  inputsAndOutputs.getFilenameOfFirstInput());
 
   if (!FileBufOrErr) {
-    Instance.getASTContext().Diags.diagnose(
-        SourceLoc(), diag::error_open_input_file,
-        inputsAndOutputs.getFilenameOfFirstInput(),
-        FileBufOrErr.getError().message());
+    Instance.getDiags().diagnose(SourceLoc(), diag::error_open_input_file,
+                                 inputsAndOutputs.getFilenameOfFirstInput(),
+                                 FileBufOrErr.getError().message());
     return true;
   }
   llvm::MemoryBuffer *MainFile = FileBufOrErr.get().get();
@@ -806,9 +805,9 @@ static bool compileLLVMIR(CompilerInstance &Instance) {
   if (!Module) {
     // TODO: Translate from the diagnostic info to the SourceManager location
     // if available.
-    Instance.getASTContext().Diags.diagnose(
-        SourceLoc(), diag::error_parse_input_file,
-        inputsAndOutputs.getFilenameOfFirstInput(), Err.getMessage());
+    Instance.getDiags().diagnose(SourceLoc(), diag::error_parse_input_file,
+                                 inputsAndOutputs.getFilenameOfFirstInput(),
+                                 Err.getMessage());
     return true;
   }
   return performLLVM(Invocation.getIRGenOptions(), Instance.getASTContext(),
@@ -927,7 +926,7 @@ static void emitReferenceDependenciesForAllPrimaryInputsIfNeeded(
   if (Invocation.getFrontendOptions()
           .InputsAndOutputs.hasReferenceDependenciesPath() &&
       Instance.getPrimarySourceFiles().empty()) {
-    Instance.getASTContext().Diags.diagnose(
+    Instance.getDiags().diagnose(
         SourceLoc(), diag::emit_reference_dependencies_without_primary_file);
     return;
   }
@@ -936,48 +935,47 @@ static void emitReferenceDependenciesForAllPrimaryInputsIfNeeded(
         Invocation.getReferenceDependenciesFilePathForPrimary(
             SF->getFilename());
     if (!referenceDependenciesFilePath.empty()) {
-      auto LangOpts = Invocation.getLangOptions();
+      const auto LangOpts = Invocation.getLangOptions();
       (void)fine_grained_dependencies::emitReferenceDependencies(
-          Instance.getASTContext().Diags, SF,
-          *Instance.getDependencyTracker(),
+          Instance.getDiags(), SF, *Instance.getDependencyTracker(),
           referenceDependenciesFilePath,
           LangOpts.EmitFineGrainedDependencySourcefileDotFiles);
     }
   }
 }
 static void
-emitSwiftRangesForAllPrimaryInputsIfNeeded(const CompilerInstance &Instance) {
+emitSwiftRangesForAllPrimaryInputsIfNeeded(CompilerInstance &Instance) {
   const auto &Invocation = Instance.getInvocation();
   if (Invocation.getFrontendOptions().InputsAndOutputs.hasSwiftRangesPath() &&
       Instance.getPrimarySourceFiles().empty()) {
-    Instance.getASTContext().Diags.diagnose(
-        SourceLoc(), diag::emit_swift_ranges_without_primary_file);
+    Instance.getDiags().diagnose(SourceLoc(),
+                                 diag::emit_swift_ranges_without_primary_file);
     return;
   }
   for (auto *SF : Instance.getPrimarySourceFiles()) {
     const std::string &swiftRangesFilePath =
         Invocation.getSwiftRangesFilePathForPrimary(SF->getFilename());
     if (!swiftRangesFilePath.empty()) {
-      (void)Instance.emitSwiftRanges(Instance.getASTContext().Diags, SF,
+      (void)Instance.emitSwiftRanges(Instance.getDiags(), SF,
                                      swiftRangesFilePath);
     }
   }
 }
 static void emitCompiledSourceForAllPrimaryInputsIfNeeded(
-    const CompilerInstance &Instance) {
+    CompilerInstance &Instance) {
   const auto &Invocation = Instance.getInvocation();
   if (Invocation.getFrontendOptions()
           .InputsAndOutputs.hasCompiledSourcePath() &&
       Instance.getPrimarySourceFiles().empty()) {
-    Instance.getASTContext().Diags.diagnose(
+    Instance.getDiags().diagnose(
         SourceLoc(), diag::emit_compiled_source_without_primary_file);
     return;
   }
   for (auto *SF : Instance.getPrimarySourceFiles()) {
     const std::string &compiledSourceFilePath =
         Invocation.getCompiledSourceFilePathForPrimary(SF->getFilename());
     if (!compiledSourceFilePath.empty()) {
-      (void)Instance.emitCompiledSource(Instance.getASTContext().Diags, SF,
+      (void)Instance.emitCompiledSource(Instance.getDiags(), SF,
                                         compiledSourceFilePath);
     }
   }
@@ -1037,9 +1035,8 @@ static bool writeLdAddCFileIfNeeded(CompilerInstance &Instance) {
   std::error_code EC;
   llvm::raw_fd_ostream OS(Path, EC, llvm::sys::fs::F_None);
   if (EC) {
-    module->getASTContext().Diags.diagnose(SourceLoc(),
-                                           diag::error_opening_output,
-                                           Path, EC.message());
+    Instance.getDiags().diagnose(SourceLoc(), diag::error_opening_output, Path,
+                                 EC.message());
     return true;
   }
   OS << "// Automatically generated C source file from the Swift compiler \n"
@@ -1254,7 +1251,7 @@ static bool performCompile(CompilerInstance &Instance,
     scanDependencies(Instance);
   }
 
-  (void)emitMakeDependenciesIfNeeded(Context.Diags,
+  (void)emitMakeDependenciesIfNeeded(Instance.getDiags(),
                                      Instance.getDependencyTracker(), opts);
 
   if (Action == FrontendOptions::ActionType::ResolveImports ||
@@ -2187,6 +2184,14 @@ int swift::performFrontend(ArrayRef<const char *> Args,
                        Invocation.getFrontendOptions().DumpAPIPath);
   }
 
+  // If we're asked to enable private intransitive dependencies, we need to
+  // write over the dependency files we just emitted because we need to
+  // get the dependencies written post-Sema down on disk.
+  // FIXME: Evaluate the impact turning this on universally has.
+  if (Invocation.getLangOptions().EnableExperientalPrivateIntransitiveDependencies) {
+    emitReferenceDependenciesForAllPrimaryInputsIfNeeded(*Instance);
+  }
+
   // Verify reference dependencies of the current compilation job *before*
   // verifying diagnostics so that the former can be tested via the latter.
   if (Invocation.getFrontendOptions().EnableIncrementalDependencyVerifier) {

test/Incremental/PrivateDependencies/Inputs/InterestingType.swift

@@ -0,0 +1,23 @@
+protocol IntMaker {}
+extension IntMaker {
+  func make() -> Int { return 0 }
+}
+
+protocol DoubleMaker {}
+extension DoubleMaker {
+  func make() -> Double { return 0 }
+}
+
+
+#if OLD
+typealias InterestingType = Int
+typealias InterestingProto = IntMaker
+
+#elseif NEW
+typealias InterestingType = Double
+typealias InterestingProto = DoubleMaker
+
+#else
+typealias InterestingType = ErrorMustSetOLDOrNew
+
+#endif

test/Incremental/PrivateDependencies/private-function-fine.swift

@@ -0,0 +1,19 @@
+// REQUIRES: shell
+// Also uses awk:
+// XFAIL OS=windows
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DOLD -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-OLD
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DNEW -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-NEW
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+private func testParamType(_: InterestingType) {}
+
+// CHECK-OLD: sil_global hidden @$s4main1x{{[^ ]+}} : ${{(@[a-zA-Z_]+ )?}}(Int) -> ()
+// CHECK-NEW: sil_global hidden @$s4main1x{{[^ ]+}} : ${{(@[a-zA-Z_]+ )?}}(Double) -> ()
+internal var x = testParamType
+
+// CHECK-DEPS: topLevel interface  '' InterestingType false

test/Incremental/PrivateDependencies/private-function-return-type-fine.swift

@@ -0,0 +1,19 @@
+// REQUIRES: shell
+// Also uses awk:
+// XFAIL OS=windows
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DOLD -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-OLD
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DNEW -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-NEW
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+private func testReturnType() -> InterestingType { fatalError() }
+
+// CHECK-OLD: sil_global @$s4main1x{{[^ ]+}} : $Int
+// CHECK-NEW: sil_global @$s4main1x{{[^ ]+}} : $Double
+public var x = testReturnType() + 0
+
+// CHECK-DEPS: topLevel interface  '' InterestingType false

test/Incremental/PrivateDependencies/private-protocol-conformer-ext-fine.swift

@@ -0,0 +1,24 @@
+// REQUIRES: shell
+// Also uses awk:
+// XFAIL OS=windows
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DOLD -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-OLD
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DNEW -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-NEW
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+private struct Test {}
+extension Test : InterestingProto {}
+
+// CHECK-OLD: sil_global @$s4main1x{{[^ ]+}} : $Int
+// CHECK-NEW: sil_global @$s4main1x{{[^ ]+}} : $Double
+public var x = Test().make() + 0
+
+// CHECK-DEPS-DAG: topLevel interface  '' InterestingProto false
+
+// CHECK-DEPS-DAG: member interface  4main{{8IntMaker|11DoubleMaker}}P make false
+
+// CHECK-DEPS-DAG: nominal interface  4main{{8IntMaker|11DoubleMaker}}P '' false