Merge pull request #23866 from harlanhaskins/shock-and-awe-5.1

[5.1] [ParseableInterfaces] Short-circuit module loading in the PIML

Author: Harlan Haskins

include/swift/Serialization/SerializedModuleLoader.h

@@ -51,6 +51,18 @@ class SerializedModuleLoaderBase : public ModuleLoader {
                   std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
                   bool &isFramework);
 
+  /// Attempts to search the provided directory for a loadable serialized
+  /// .swiftmodule with the provided `ModuleFilename`. Subclasses must
+  /// override this method to perform their custom module lookup behavior.
+  ///
+  /// If such a module could not be loaded, the subclass must return a
+  /// `std::error_code` indicating the failure. There are two specific error
+  /// codes that will be treated specially:
+  /// - `errc::no_such_file_or_directory`: The module loader will stop looking
+  ///   for loadable modules and will diagnose the lookup failure.
+  /// - `errc::not_supported`: The module loader will stop looking for loadable
+  ///   modules and will defer to the remaining module loaders to look up this
+  ///   module.
   virtual std::error_code findModuleFilesInDirectory(
       AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename,
       StringRef ModuleDocFilename,

lib/Frontend/ParseableInterfaceModuleLoader.cpp

@@ -864,13 +864,25 @@ class ParseableInterfaceModuleLoaderImpl {
       if (isForwardingModule) {
         if (auto forwardingModule = ForwardingModule::load(*buf)) {
           std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
-          if (forwardingModuleIsUpToDate(*forwardingModule, deps, moduleBuffer))
+          if (forwardingModuleIsUpToDate(*forwardingModule, deps,
+                                         moduleBuffer)) {
+            LLVM_DEBUG(llvm::dbgs() << "Found up-to-date forwarding module at "
+                                    << cachedOutputPath << "\n");
             return DiscoveredModule::forwarded(
               forwardingModule->underlyingModulePath, std::move(moduleBuffer));
+          }
+
+          LLVM_DEBUG(llvm::dbgs() << "Found out-of-date forwarding module at "
+                     << cachedOutputPath << "\n");
         }
       // Otherwise, check if the AST buffer itself is up to date.
       } else if (serializedASTBufferIsUpToDate(*buf, deps)) {
+        LLVM_DEBUG(llvm::dbgs() << "Found up-to-date cached module at "
+                                << cachedOutputPath << "\n");
         return DiscoveredModule::normal(cachedOutputPath, std::move(buf));
+      } else {
+        LLVM_DEBUG(llvm::dbgs() << "Found out-of-date cached module at "
+                   << cachedOutputPath << "\n");
       }
     }
 
@@ -883,23 +895,39 @@ class ParseableInterfaceModuleLoaderImpl {
       llvm::SmallString<256> scratch;
       std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
       auto path = computePrebuiltModulePath(scratch);
-      if (path && swiftModuleIsUpToDate(*path, deps, moduleBuffer))
-        return DiscoveredModule::prebuilt(*path, std::move(moduleBuffer));
+      if (path) {
+        if (swiftModuleIsUpToDate(*path, deps, moduleBuffer)) {
+          LLVM_DEBUG(llvm::dbgs() << "Found up-to-date prebuilt module at "
+                                  << path->str() << "\n");
+          return DiscoveredModule::prebuilt(*path, std::move(moduleBuffer));
+        } else {
+          LLVM_DEBUG(llvm::dbgs() << "Found out-of-date prebuilt module at "
+                     << modulePath << "\n");
+        }
+      }
     }
 
     // Finally, if there's a module adjacent to the .swiftinterface that we can
     // _likely_ load (it validates OK and is up to date), bail early with
     // errc::not_supported, so the next (serialized) loader in the chain will
-    // load it. Alternately, if there's a .swiftmodule present but we can't even
+    // load it.
+    // Alternately, if there's a .swiftmodule present but we can't even
     // read it (for whatever reason), we should let the other module loader
     // diagnose it.
     if (!shouldLoadAdjacentModule)
       return notFoundError;
 
     auto adjacentModuleBuffer = fs.getBufferForFile(modulePath);
     if (adjacentModuleBuffer) {
-      if (serializedASTBufferIsUpToDate(*adjacentModuleBuffer.get(), deps))
+      if (serializedASTBufferIsUpToDate(*adjacentModuleBuffer.get(), deps)) {
+        LLVM_DEBUG(llvm::dbgs() << "Found up-to-date module at "
+                                << modulePath
+                                << "; deferring to serialized module loader\n");
         return std::make_error_code(std::errc::not_supported);
+      } else {
+        LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module at "
+                                << modulePath << "\n");
+      }
     } else if (adjacentModuleBuffer.getError() != notFoundError) {
       return std::make_error_code(std::errc::not_supported);
     }
@@ -1053,8 +1081,15 @@ std::error_code ParseableInterfaceModuleLoader::findModuleFilesInDirectory(
   auto Ext = file_types::getExtension(file_types::TY_SwiftParseableInterfaceFile);
   InPath = ModPath;
   path::replace_extension(InPath, Ext);
-  if (!fs.exists(InPath))
+  if (!fs.exists(InPath)) {
+    if (fs.exists(ModPath)) {
+      LLVM_DEBUG(llvm::dbgs()
+        << "No .swiftinterface file found adjacent to module file "
+        << ModPath.str() << "\n");
+      return std::make_error_code(std::errc::not_supported);
+    }
     return std::make_error_code(std::errc::no_such_file_or_directory);
+  }
 
   // Create an instance of the Impl to do the heavy lifting.
   ParseableInterfaceModuleLoaderImpl Impl(

lib/Serialization/SerializedModuleLoader.cpp

@@ -218,6 +218,8 @@ SerializedModuleLoaderBase::findModule(AccessPathElem moduleID,
                         moduleBuffer, moduleDocBuffer);
       if (!result) {
         return true;
+      } else if (result == std::errc::not_supported) {
+        return false;
       } else if (result != std::errc::no_such_file_or_directory) {
         return None;
       }
@@ -251,6 +253,8 @@ SerializedModuleLoaderBase::findModule(AccessPathElem moduleID,
                                                moduleBuffer, moduleDocBuffer);
       if (!result)
         return true;
+      else if (result == std::errc::not_supported)
+        return false;
     }
   }
 
@@ -320,6 +324,8 @@ SerializedModuleLoaderBase::findModule(AccessPathElem moduleID,
                                                moduleBuffer, moduleDocBuffer);
       if (!result)
         return true;
+      else if (result == std::errc::not_supported)
+        return false;
     }
   }
 

test/ParseableInterface/ModuleCache/prefer-local-module-to-sdk.swift

@@ -0,0 +1,24 @@
+// RUN: %empty-directory(%t/BuildDir)
+// RUN: %empty-directory(%t/SecondBuildDir/Lib.swiftmodule)
+// RUN: %empty-directory(%t/ModuleCache)
+
+// RUN: echo 'public func showsUpInBothPlaces() {}' > %t/Lib.swift
+
+// 1. Create a .swiftinterface file containing just one API, and put it inside a second build dir (without a .swiftmodule)
+// RUN: %target-swift-frontend -typecheck %t/Lib.swift -emit-parseable-module-interface-path %t/SecondBuildDir/Lib.swiftmodule/%target-cpu.swiftinterface
+
+// 2. Add a new API to the module, and compile just the serialized version in the build dir.
+// RUN: echo 'public func onlyInTheCompiledModule() {}' >> %t/Lib.swift
+// RUN: %target-swift-frontend -emit-module %t/Lib.swift -o %t/BuildDir/Lib.swiftmodule -emit-parseable-module-interface-path %t/BuildDir/Lib.swiftinterface
+
+// 3. Make sure when we compile this test file, we can access both APIs since we'll
+//    load the compiled .swiftmodule instead of the .swiftinterface in the SDK.
+// RUN: %target-swift-frontend -typecheck %s -I %t/BuildDir -I %t/SecondBuildDir -module-cache-path %t/ModuleCache
+
+// 4. Make sure we didn't compile any .swiftinterfaces into the module cache.
+// RUN: ls %t/ModuleCache | not grep 'swiftmodule'
+
+import Lib
+
+showsUpInBothPlaces()
+onlyInTheCompiledModule()