[ScanDependency] Move binary module validation into scanner

Improve swift dependency scanner by validating and selecting dependency
module into scanner. This provides benefits that:
* Build system does not need to schedule interface compilation task if
  the candidate module is picked, it can just use the candidate module
  directly.
* There is no need for forwarding module in the explicit module build.
  Since the build system is coordinating the build, there is no need for
  the forwarding module in the module cache to avoid duplicated work,
* This also correctly supports all the module loading modes in the
  dependency scanner.

This is achieved by only adding validate and up-to-date binary module as
the candidate module for swift interface module dependency. This allows
caching build to construct the correct dependency in the CAS. If there
is a candidate module for the interface module, dependency scanner will
return a binary module dependency in the dependency graph.

The legacy behavior is preserved with a hidden frontend flag
`-legacy-scanner-behavior`, while the output is mostly interchangeable
with new scanner behavior with `prefer-interface` module loading mode
except the candidate module will not be returned.

rdar://123711823

Author: cachemeifyoucan

include/swift/AST/ModuleDependencies.h

@@ -1115,6 +1115,12 @@ class ModuleDependenciesCache {
   std::optional<const ModuleDependencyInfo *>
   findDependency(StringRef moduleName) const;
 
+  /// Look for known existing dependencies.
+  ///
+  /// \returns the cached result.
+  const ModuleDependencyInfo &
+  findKnownDependency(const ModuleDependencyID &moduleID) const;
+
   /// Record dependencies for the given module.
   void recordDependency(StringRef moduleName,
                         ModuleDependencyInfo dependencies);

include/swift/AST/ModuleLoader.h

@@ -190,8 +190,8 @@ struct SubCompilerInstanceInfo {
 /// Abstract interface for a checker of module interfaces and prebuilt modules.
 class ModuleInterfaceChecker {
 public:
-  virtual std::vector<std::string>
-  getCompiledModuleCandidatesForInterface(StringRef moduleName,
+  virtual std::optional<std::string>
+  getCompiledModuleCandidateForInterface(StringRef moduleName,
                                           StringRef interfacePath) = 0;
 
   /// Given a list of potential ready-to-use compiled modules for \p interfacePath,

include/swift/AST/SearchPathOptions.h

@@ -38,6 +38,15 @@ enum class ModuleSearchPathKind {
   RuntimeLibrary,
 };
 
+/// Specifies how to load modules when both a module interface and serialized
+/// AST are present, or whether to disallow one format or the other altogether.
+enum class ModuleLoadingMode {
+  PreferInterface,
+  PreferSerialized,
+  OnlyInterface,
+  OnlySerialized
+};
+
 /// A single module search path that can come from different sources, e.g.
 /// framework search paths, import search path etc.
 class ModuleSearchPath : public llvm::RefCountedBase<ModuleSearchPath> {
@@ -499,6 +508,12 @@ class SearchPathOptions {
   /// original form.
   PathObfuscator DeserializedPathRecoverer;
 
+  /// Specify the module loading behavior of the compilation.
+  ModuleLoadingMode ModuleLoadMode = ModuleLoadingMode::PreferSerialized;
+
+  /// Legacy scanner search behavior.
+  bool UseLegacyScannerBehavior = false;
+
   /// Return all module search paths that (non-recursively) contain a file whose
   /// name is in \p Filenames.
   SmallVector<const ModuleSearchPath *, 4>
@@ -546,7 +561,9 @@ class SearchPathOptions {
                         RuntimeResourcePath,
                         hash_combine_range(RuntimeLibraryImportPaths.begin(),
                                            RuntimeLibraryImportPaths.end()),
-                        DisableModulesValidateSystemDependencies);
+                        DisableModulesValidateSystemDependencies,
+                        UseLegacyScannerBehavior,
+                        ModuleLoadMode);
   }
 
   /// Return a hash code of any components from these options that should

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -527,8 +527,8 @@ class ModuleInterfaceCheckerImpl: public ModuleInterfaceChecker {
                                 RequireNoncopyableGenerics_t requireNCGenerics):
     ModuleInterfaceCheckerImpl(Ctx, cacheDir, prebuiltCacheDir, StringRef(),
                                opts, requiresOSSAModules, requireNCGenerics) {}
-  std::vector<std::string>
-  getCompiledModuleCandidatesForInterface(StringRef moduleName,
+  std::optional<std::string>
+  getCompiledModuleCandidateForInterface(StringRef moduleName,
                                           StringRef interfacePath) override;
 
   /// Given a list of potential ready-to-use compiled modules for \p interfacePath,

include/swift/Option/FrontendOptions.td

@@ -1332,4 +1332,10 @@ def disable_experimental_parser_round_trip : Flag<["-"],
   "disable-experimental-parser-round-trip">,
   HelpText<"Disable round trip through the new swift parser">;
 
+def legacy_scanner_behavior: Flag<["-"], "legacy-scanner-behavior">,
+    HelpText<"Use legacy scanner behavior">;
+def module_load_mode: Separate<["-"], "module-load-mode">,
+    MetaVarName<"only-interface|prefer-interface|prefer-serialized|only-serialized">,
+    HelpText<"Module loading mode">;
+
 } // end let Flags = [FrontendOption, NoDriverOption, HelpHidden]

include/swift/Serialization/SerializedModuleLoader.h

@@ -16,6 +16,7 @@
 #include "swift/AST/FileUnit.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/ModuleLoader.h"
+#include "swift/AST/SearchPathOptions.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/PrefixMapper.h"
 #include "llvm/TargetParser/Triple.h"
@@ -28,15 +29,6 @@ namespace file_types {
   enum ID : uint8_t;
 }
 
-/// Specifies how to load modules when both a module interface and serialized
-/// AST are present, or whether to disallow one format or the other altogether.
-enum class ModuleLoadingMode {
-  PreferInterface,
-  PreferSerialized,
-  OnlyInterface,
-  OnlySerialized
-};
-
 /// How a dependency should be loaded.
 ///
 /// \sa getTransitiveLoadingBehavior

lib/AST/ModuleDependencies.cpp

@@ -736,6 +736,13 @@ ModuleDependenciesCache::findDependency(StringRef moduleName) const {
   return std::nullopt;
 }
 
+const ModuleDependencyInfo &ModuleDependenciesCache::findKnownDependency(
+    const ModuleDependencyID &moduleID) const {
+  auto dep = findDependency(moduleID);
+  assert(dep && "dependency unknown");
+  return **dep;
+}
+
 bool ModuleDependenciesCache::hasDependency(const ModuleDependencyID &moduleID) const {
   return hasDependency(moduleID.ModuleName, moduleID.Kind);
 }

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -175,7 +175,8 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
       ScanASTContext,
       *static_cast<ModuleInterfaceCheckerImpl *>(
           ScanASTContext.getModuleInterfaceChecker()),
-      &DependencyTracker, ModuleLoadingMode::OnlyInterface);
+      &DependencyTracker,
+      ScanCompilerInvocation.getSearchPathOptions().ModuleLoadMode);
 }
 
 ModuleDependencyVector