[ModuleInterface] Write, read-back and test deps stored in .sid files.

Author: graydon

include/swift/Frontend/ParseableInterfaceSupport.h

@@ -64,9 +64,10 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
   std::string CacheDir;
 
   void
-  configureSubInvocationAndOutputPath(CompilerInvocation &SubInvocation,
-                                      StringRef InPath,
-                                      llvm::SmallString<128> &OutPath);
+  configureSubInvocationAndOutputPaths(CompilerInvocation &SubInvocation,
+                                       StringRef InPath,
+                                       llvm::SmallString<128> &OutPath,
+                                       llvm::SmallString<128> &DepPath);
 
   std::error_code
   openModuleFiles(StringRef DirName, StringRef ModuleFilename,

lib/Frontend/ParseableInterfaceSupport.cpp

@@ -14,6 +14,7 @@
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticsFrontend.h"
+#include "swift/AST/FileSystem.h"
 #include "swift/AST/Module.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Frontend/ParseableInterfaceSupport.h"
@@ -32,6 +33,7 @@ using namespace swift;
 
 #define SWIFT_TOOLS_VERSION_KEY "swift-tools-version"
 #define SWIFT_MODULE_FLAGS_KEY "swift-module-flags"
+#define SWIFT_INTERFACE_DEPS_VERSION "swift-interface-deps-version-1"
 
 static bool
 extractSwiftInterfaceVersionAndArgs(DiagnosticEngine &Diags,
@@ -100,10 +102,11 @@ std::string getCacheHash(ASTContext &Ctx,
 }
 
 void
-ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPath(
+ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPaths(
     CompilerInvocation &SubInvocation,
     StringRef InPath,
-    llvm::SmallString<128> &OutPath) {
+    llvm::SmallString<128> &OutPath,
+    llvm::SmallString<128> &DepPath) {
 
   auto &SearchPathOpts = Ctx.SearchPathOpts;
   auto &LangOpts = Ctx.LangOpts;
@@ -125,8 +128,11 @@ ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPath(
   OutPath.append("-");
   OutPath.append(getCacheHash(Ctx, SubInvocation, InPath));
   OutPath.append(".");
-  auto Ext = file_types::getExtension(file_types::TY_SwiftModuleFile);
-  OutPath.append(Ext);
+  DepPath = OutPath;
+  auto OutExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
+  OutPath.append(OutExt);
+  auto DepExt = file_types::getExtension(file_types::TY_SwiftParseableInterfaceDeps);
+  DepPath.append(DepExt);
 
   auto &FEOpts = SubInvocation.getFrontendOptions();
   FEOpts.RequestedAction = FrontendOptions::ActionType::EmitModuleOnly;
@@ -137,24 +143,67 @@ ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPath(
   FEOpts.InputsAndOutputs.setMainAndSupplementaryOutputs({MainOut}, {SOPs});
 }
 
-// FIXME: this needs to be a more extensive up-to-date check.
+// Write the world's simplest dependencies file: a version identifier on
+// a line followed by a list of files, one per line.
+static bool writeSwiftInterfaceDeps(DiagnosticEngine &Diags,
+                                    ArrayRef<std::string> Deps,
+                                    StringRef DepPath) {
+  return withOutputFile(Diags, DepPath, [&](llvm::raw_pwrite_stream &out) {
+      out << SWIFT_INTERFACE_DEPS_VERSION << '\n';
+      for (auto const &D : Deps) {
+        out << D << '\n';
+      }
+      return false;
+    });
+}
+
+// Check that the output .swiftmodule file is at least as new as all the
+// dependencies it read when it was built last time.
 static bool
 swiftModuleIsUpToDate(clang::vfs::FileSystem &FS,
-                      StringRef InPath, StringRef OutPath) {
-  if (FS.exists(OutPath)) {
-    auto InStatus = FS.status(InPath);
-    auto OutStatus = FS.status(OutPath);
-    if (InStatus && OutStatus) {
-      return InStatus.get().getLastModificationTime() <=
-        OutStatus.get().getLastModificationTime();
+                      StringRef InPath, StringRef OutPath, StringRef DepPath) {
+
+  if (!FS.exists(OutPath) || !FS.exists(DepPath))
+    return false;
+
+  auto OutStatus = FS.status(OutPath);
+  if (!OutStatus)
+    return false;
+
+  auto DepBuf = FS.getBufferForFile(DepPath);
+  if (!DepBuf)
+    return false;
+
+  // Split the deps file into a vector of lines.
+  StringRef Deps =  DepBuf.get()->getBuffer();
+  SmallVector<StringRef, 16> AllDeps;
+  Deps.split(AllDeps, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+
+  // First line in vector is a version-string; check it is the expected value.
+  if (AllDeps.size() < 1 ||
+      AllDeps[0] != SWIFT_INTERFACE_DEPS_VERSION) {
+    return false;
+  }
+
+  // Overwrite the version-string entry in the vector with the .swiftinterface
+  // input file we're reading, then stat() every entry in the vector and check
+  // none are newer than the .swiftmodule (OutStatus).
+  AllDeps[0] = InPath;
+  for (auto In : AllDeps) {
+    auto InStatus = FS.status(In);
+    if (!InStatus ||
+        (InStatus.get().getLastModificationTime() >
+         OutStatus.get().getLastModificationTime())) {
+      return false;
     }
   }
-  return false;
+  return true;
 }
 
 static bool buildSwiftModuleFromSwiftInterface(
     clang::vfs::FileSystem &FS, DiagnosticEngine &Diags,
-    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath) {
+    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath,
+    StringRef DepPath) {
   bool SubError = false;
   bool RunSuccess = llvm::CrashRecoveryContext().RunSafelyOnThread([&] {
 
@@ -178,6 +227,7 @@ static bool buildSwiftModuleFromSwiftInterface(
     // module-serialization task we're trying to do here.
     LLVM_DEBUG(llvm::dbgs() << "Setting up instance\n");
     CompilerInstance SubInstance;
+    SubInstance.createDependencyTracker(/*TrackSystemDeps=*/false);
     if (SubInstance.setup(SubInvocation)) {
       SubError = true;
       return;
@@ -212,6 +262,12 @@ static bool buildSwiftModuleFromSwiftInterface(
       });
     SILMod->serialize();
     SubError = Diags.hadAnyError();
+
+    if (!SubError) {
+      SubError |= writeSwiftInterfaceDeps(
+          Diags, SubInstance.getDependencyTracker()->getDependencies(),
+          DepPath);
+    }
   });
   return !RunSuccess || SubError;
 }
@@ -227,7 +283,7 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
 
   auto &FS = *Ctx.SourceMgr.getFileSystem();
   auto &Diags = Ctx.Diags;
-  llvm::SmallString<128> InPath, OutPath;
+  llvm::SmallString<128> InPath, OutPath, DepPath;
 
   // First check to see if the .swiftinterface exists at all. Bail if not.
   InPath = DirName;
@@ -240,12 +296,12 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
   // Set up a _potential_ sub-invocation to consume the .swiftinterface and emit
   // the .swiftmodule.
   CompilerInvocation SubInvocation;
-  configureSubInvocationAndOutputPath(SubInvocation, InPath, OutPath);
+  configureSubInvocationAndOutputPaths(SubInvocation, InPath, OutPath, DepPath);
 
   // Evaluate if we need to run this sub-invocation, and if so run it.
-  if (!swiftModuleIsUpToDate(FS, InPath, OutPath)) {
+  if (!swiftModuleIsUpToDate(FS, InPath, OutPath, DepPath)) {
     if (buildSwiftModuleFromSwiftInterface(FS, Diags, SubInvocation, InPath,
-                                           OutPath))
+                                           OutPath, DepPath))
       return std::make_error_code(std::errc::invalid_argument);
   }
 

test/ParseableInterface/client.swift

@@ -3,11 +3,32 @@
 // RUN: %target-swift-frontend -enable-resilience -emit-parseable-module-interface-path %t/TestModule.swiftinterface -module-name TestModule %S/Inputs/other.swift -emit-module -o /dev/null
 // RUN: test -f %t/TestModule.swiftinterface
 // RUN: %target-swift-frontend -typecheck -enable-parseable-module-interface -module-cache-path %t/modulecache -I %t %s
+
+// RUN: test ! %t/modulecache/TestModule-*.swiftmodule -ot %t/TestModule.swiftinterface
 // RUN: llvm-bcanalyzer -dump %t/modulecache/TestModule-*.swiftmodule | %FileCheck %s -check-prefix=CHECK-SWIFTMODULE
+// RUN: %FileCheck %s -check-prefix=CHECK-SID <%t/modulecache/TestModule-*.sid
+
+// Now add a "dependency" to the .sid file but set it to be quite old
+// RUN: echo %t/fake-dep >>%t/modulecache/TestModule-*.sid
+// RUN: touch -t 201401240005 %t/fake-dep
+// RUN: %FileCheck %s -check-prefix=CHECK-SID2 <%t/modulecache/TestModule-*.sid
+
+// Check that the cache does not rebuild
+// RUN: %target-swift-frontend -typecheck -enable-parseable-module-interface -module-cache-path %t/modulecache -I %t %s
+// RUN: %FileCheck %s -check-prefix=CHECK-SID2 <%t/modulecache/TestModule-*.sid
+
+// Now touch the dependency a ways into the future, and check that the cache _does_ rebuild
+// (Sorry, this test will need refreshing in the year 2035)
+// RUN: touch -t 203501240005 %t/fake-dep
+// RUN: %target-swift-frontend -typecheck -enable-parseable-module-interface -module-cache-path %t/modulecache -I %t %s
+// RUN: %FileCheck %s -check-prefix=CHECK-SID3 <%t/modulecache/TestModule-*.sid
 
 // CHECK-SWIFTMODULE: {{MODULE_NAME.*blob data = 'TestModule'}}
 // CHECK-SWIFTMODULE: FUNC_DECL
 // CHECK-SWIFTMODULE: RESILIENCE_STRATEGY
+// CHECK-SID: Swift.swiftmodule
+// CHECK-SID2: fake-dep
+// CHECK-SID3-NOT: fake-dep
 
 import TestModule
 func foo() {