[ScanDependency] Move binary module validation into scanner

Improve swift dependency scanner by validating and selecting dependency
module into scanner. This provides benefits that:
* Build system does not need to schedule interface compilation task if
  the candidate module is picked, it can just use the candidate module
  directly.
* There is no need for forwarding module in the explicit module build.
  Since the build system is coordinating the build, there is no need for
  the forwarding module in the module cache to avoid duplicated work,
* This also correctly supports all the module loading modes in the
  dependency scanner.

This is achieved by only adding validate and up-to-date binary module as
the candidate module for swift interface module dependency. This allows
caching build to construct the correct dependency in the CAS. If there
is a candidate module for the interface module, dependency scanner will
return a binary module dependency in the dependency graph.

The legacy behavior is mostly preserved with a hidden frontend flag
`-no-scanner-module-validation`, while the scanner output is mostly
interchangeable with new scanner behavior with `prefer-interface` module
loading mode except the candidate module will not be returned.

rdar://123711823
(cherry picked from commit 0e12f20)

Author: cachemeifyoucan

include/swift/AST/ModuleDependencies.h

@@ -1097,6 +1097,12 @@ class ModuleDependenciesCache {
   std::optional<const ModuleDependencyInfo *>
   findDependency(StringRef moduleName) const;
 
+  /// Look for known existing dependencies.
+  ///
+  /// \returns the cached result.
+  const ModuleDependencyInfo &
+  findKnownDependency(const ModuleDependencyID &moduleID) const;
+
   /// Record dependencies for the given module.
   void recordDependency(StringRef moduleName,
                         ModuleDependencyInfo dependencies);

include/swift/AST/SearchPathOptions.h

@@ -38,6 +38,15 @@ enum class ModuleSearchPathKind {
   RuntimeLibrary,
 };
 
+/// Specifies how to load modules when both a module interface and serialized
+/// AST are present, or whether to disallow one format or the other altogether.
+enum class ModuleLoadingMode {
+  PreferInterface,
+  PreferSerialized,
+  OnlyInterface,
+  OnlySerialized
+};
+
 /// A single module search path that can come from different sources, e.g.
 /// framework search paths, import search path etc.
 class ModuleSearchPath : public llvm::RefCountedBase<ModuleSearchPath> {
@@ -499,6 +508,12 @@ class SearchPathOptions {
   /// original form.
   PathObfuscator DeserializedPathRecoverer;
 
+  /// Specify the module loading behavior of the compilation.
+  ModuleLoadingMode ModuleLoadMode = ModuleLoadingMode::PreferSerialized;
+
+  /// Legacy scanner search behavior.
+  bool NoScannerModuleValidation = false;
+
   /// Return all module search paths that (non-recursively) contain a file whose
   /// name is in \p Filenames.
   SmallVector<const ModuleSearchPath *, 4>
@@ -546,7 +561,9 @@ class SearchPathOptions {
                         RuntimeResourcePath,
                         hash_combine_range(RuntimeLibraryImportPaths.begin(),
                                            RuntimeLibraryImportPaths.end()),
-                        DisableModulesValidateSystemDependencies);
+                        DisableModulesValidateSystemDependencies,
+                        NoScannerModuleValidation,
+                        ModuleLoadMode);
   }
 
   /// Return a hash code of any components from these options that should

include/swift/Option/FrontendOptions.td

@@ -1337,4 +1337,10 @@ def disable_strict_concurrency_region_based_isolation : Flag<["-"],
   HelpText<"Disable region based isolation when running with strict concurrency enabled. Only enabled with asserts">,
   Flags<[HelpHidden]>;
 
+def no_scanner_module_validation: Flag<["-"], "no-scanner-module-validation">,
+    HelpText<"Do not validate binary modules in scanner and delegate the validation to swift-frontend">;
+def module_load_mode: Separate<["-"], "module-load-mode">,
+    MetaVarName<"only-interface|prefer-interface|prefer-serialized|only-serialized">,
+    HelpText<"Module loading mode">;
+
 } // end let Flags = [FrontendOption, NoDriverOption, HelpHidden]

include/swift/Serialization/SerializedModuleLoader.h

@@ -16,6 +16,7 @@
 #include "swift/AST/FileUnit.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/ModuleLoader.h"
+#include "swift/AST/SearchPathOptions.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/PrefixMapper.h"
 #include "llvm/TargetParser/Triple.h"
@@ -28,15 +29,6 @@ namespace file_types {
   enum ID : uint8_t;
 }
 
-/// Specifies how to load modules when both a module interface and serialized
-/// AST are present, or whether to disallow one format or the other altogether.
-enum class ModuleLoadingMode {
-  PreferInterface,
-  PreferSerialized,
-  OnlyInterface,
-  OnlySerialized
-};
-
 /// How a dependency should be loaded.
 ///
 /// \sa getTransitiveLoadingBehavior

lib/AST/ModuleDependencies.cpp

@@ -739,6 +739,13 @@ ModuleDependenciesCache::findDependency(StringRef moduleName) const {
   return std::nullopt;
 }
 
+const ModuleDependencyInfo &ModuleDependenciesCache::findKnownDependency(
+    const ModuleDependencyID &moduleID) const {
+  auto dep = findDependency(moduleID);
+  assert(dep && "dependency unknown");
+  return **dep;
+}
+
 bool ModuleDependenciesCache::hasDependency(const ModuleDependencyID &moduleID) const {
   return hasDependency(moduleID.ModuleName, moduleID.Kind);
 }

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -185,7 +185,8 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
       ScanASTContext,
       *static_cast<ModuleInterfaceCheckerImpl *>(
           ScanASTContext.getModuleInterfaceChecker()),
-      &DependencyTracker, ModuleLoadingMode::OnlyInterface);
+      &DependencyTracker,
+      ScanCompilerInvocation.getSearchPathOptions().ModuleLoadMode);
 }
 
 ModuleDependencyVector