[ClangImporter] Merge Swift & Clang VFS

The clang importer has to deal with two virtual file systems, one coming
from clang, and one coming from swift. Currently, if both are set, we
emit a diagnostic that we'll pick the swift one.

This commit changes that, by merging the two virtual file systems into a
single overlay file system, and using that. To make this possible, we
always initialize the file manager with an overlay file system. In the
clang importer, we then create a new overlay file system, starting with
the one coming from clang, and adding overlays from swift on top.

The motivation for this change is the reproducer infrastructure in LLDB,
which adds a third virtual file system to the mix.

Author: JDevlieghere

include/swift/AST/DiagnosticsClangImporter.def

@@ -91,10 +91,6 @@ WARNING(implicit_bridging_header_imported_from_module,none,
         "is deprecated and will be removed in a later version of Swift",
         (StringRef, Identifier))
 
-WARNING(clang_vfs_overlay_is_ignored,none,
-        "ignoring '-ivfsoverlay' options provided to '-Xcc' in favor of "
-        "'-vfsoverlay'", ())
-
 #ifndef DIAG_NO_UNDEF
 # if defined(DIAG)
 #  undef DIAG

include/swift/Basic/SourceManager.h

@@ -23,10 +23,16 @@
 
 namespace swift {
 
+static inline llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>
+getRealOverlayFileSystem() {
+  return llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>(
+      new llvm::vfs::OverlayFileSystem(llvm::vfs::getRealFileSystem()));
+}
+
 /// This class manages and owns source buffers.
 class SourceManager {
   llvm::SourceMgr LLVMSourceMgr;
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> FileSystem;
   unsigned CodeCompletionBufferID = 0U;
   unsigned CodeCompletionOffset;
 
@@ -49,9 +55,9 @@ class SourceManager {
   mutable std::pair<const char *, const VirtualFile*> CachedVFile = {nullptr, nullptr};
 
 public:
-  SourceManager(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
-                    llvm::vfs::getRealFileSystem())
-    : FileSystem(FS) {}
+  SourceManager(llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> FS =
+                    getRealOverlayFileSystem())
+      : FileSystem(FS) {}
 
   llvm::SourceMgr &getLLVMSourceMgr() {
     return LLVMSourceMgr;
@@ -60,11 +66,12 @@ class SourceManager {
     return LLVMSourceMgr;
   }
 
-  void setFileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
+  void
+  setFileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> FS) {
     FileSystem = FS;
   }
 
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getFileSystem() {
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> getFileSystem() {
     return FileSystem;
   }
 
@@ -254,4 +261,3 @@ class SourceManager {
 } // end namespace swift
 
 #endif // SWIFT_BASIC_SOURCEMANAGER_H
-

lib/ClangImporter/ClangImporter.cpp

@@ -1012,14 +1012,35 @@ ClangImporter::create(ASTContext &ctx,
 
   // Set up the file manager.
   {
-    if (!ctx.SearchPathOpts.VFSOverlayFiles.empty()) {
-      // If the clang instance has overlays it means the user has provided
-      // -ivfsoverlay options and swift -vfsoverlay options.  We're going to
-      // clobber their file system with our own, so warn about it.
-      if (!instance.getHeaderSearchOpts().VFSOverlayFiles.empty()) {
-        ctx.Diags.diagnose(SourceLoc(), diag::clang_vfs_overlay_is_ignored);
-      }
+    if (instance.getHeaderSearchOpts().VFSOverlayFiles.empty()) {
       instance.setVirtualFileSystem(ctx.SourceMgr.getFileSystem());
+    } else {
+      // Initialize the clang VFS from its compiler invocation.
+      instance.createFileManager();
+
+      // Create a new overlay file system for the clang importer with the clang
+      // VFS as its root.
+      auto ClangImporterFS =
+          llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>(
+              new llvm::vfs::OverlayFileSystem(
+                  &instance.getVirtualFileSystem()));
+
+      auto SwiftFS = ctx.SourceMgr.getFileSystem();
+      auto it = SwiftFS->overlays_rbegin();
+      auto end = SwiftFS->overlays_rend();
+
+      // The Swift file system is an overlay file system with the real file
+      // system as its root. Skip the root so we query the other overlays
+      // before falling back to the real file system.
+      it++;
+
+      // Add all remaining Swift overlay file systems to the new file system.
+      while (it != end) {
+        ClangImporterFS->pushOverlay(*it);
+        it++;
+      }
+
+      instance.setVirtualFileSystem(ClangImporterFS);
     }
     instance.createFileManager();
   }

lib/Frontend/Frontend.cpp

@@ -235,9 +235,6 @@ static bool loadAndValidateVFSOverlay(
     const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &BaseFS,
     const llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> &OverlayFS,
     DiagnosticEngine &Diag) {
-  // FIXME: It should be possible to allow chained lookup of later VFS overlays
-  // through the mapping defined by earlier overlays.
-  // See rdar://problem/39440687
   auto Buffer = BaseFS->getBufferForFile(File);
   if (!Buffer) {
     Diag.diagnose(SourceLoc(), diag::cannot_open_file, File,
@@ -264,14 +261,7 @@ bool CompilerInstance::setUpVirtualFileSystemOverlays() {
     hadAnyFailure |=
         loadAndValidateVFSOverlay(File, BaseFS, OverlayFS, Diagnostics);
   }
-
-  // If we successfully loaded all the overlays, let the source manager and
-  // diagnostic engine take advantage of the overlay file system.
-  if (!hadAnyFailure &&
-      (OverlayFS->overlays_begin() != OverlayFS->overlays_end())) {
-    SourceMgr.setFileSystem(OverlayFS);
-  }
-
+  SourceMgr.setFileSystem(OverlayFS);
   return hadAnyFailure;
 }
 

lib/Frontend/ParseableInterfaceModuleLoader.cpp

@@ -273,7 +273,7 @@ static Optional<StringRef> getRelativeDepPath(StringRef DepPath,
 /// output path.
 /// \note Needs to be in the swift namespace so CompilerInvocation can see it.
 class swift::ParseableInterfaceBuilder {
-  llvm::vfs::FileSystem &fs;
+  llvm::vfs::OverlayFileSystem &fs;
   DiagnosticEngine &diags;
   const StringRef interfacePath;
   const StringRef moduleName;
@@ -765,7 +765,7 @@ class ParseableInterfaceModuleLoaderImpl {
   using AccessPathElem = std::pair<Identifier, SourceLoc>;
   friend class swift::ParseableInterfaceModuleLoader;
   ASTContext &ctx;
-  llvm::vfs::FileSystem &fs;
+  llvm::vfs::OverlayFileSystem &fs;
   DiagnosticEngine &diags;
   ModuleRebuildInfo rebuildInfo;
   const StringRef modulePath;

test/Frontend/Inputs/vfs/a-modulemap

@@ -1,3 +1,7 @@
 module VFSMappedModule {
   header "VFSMappedModule.h"
 }
+
+module YetAnotherVFSMappedModule {
+  header "YetAnotherVFSMappedModule.h"
+}

test/Frontend/Inputs/vfs/b-header

@@ -0,0 +1 @@
+#define MAJOR_SUCCESS 1

test/Frontend/Inputs/vfs/quaternary-vfsoverlay.yaml

@@ -0,0 +1,17 @@
+{
+  'version': 0,
+  'use-external-names': false,
+  'roots': [
+    {
+      'name': 'OUT_DIR', 'type': 'directory',
+      'contents': [
+        { 'name': 'YetAnotherVFSMappedModule.h', 'type': 'file',
+          'external-contents': 'INPUT_DIR/vfs/b-header'
+        },
+        { 'name': 'YetAnotherVFSMappedModule.framework/Headers/VFSMappedModule.h', 'type': 'file',
+          'external-contents': 'INPUT_DIR/vfs/b-header'
+        },
+      ]
+    },
+  ]
+}

test/Frontend/vfs.swift

@@ -2,6 +2,7 @@
 // RUN: sed -e "s|INPUT_DIR|%/S/Inputs|g" -e "s|OUT_DIR|%/t|g" %S/Inputs/vfs/vfsoverlay.yaml > %t/overlay.yaml
 // RUN: sed -e "s|INPUT_DIR|%/S/Inputs|g" -e "s|OUT_DIR|%/t|g" %S/Inputs/vfs/secondary-vfsoverlay.yaml > %t/secondary-overlay.yaml
 // RUN: sed -e "s|INPUT_DIR|%/S/Inputs|g" -e "s|OUT_DIR|%/t|g" %S/Inputs/vfs/tertiary-vfsoverlay.yaml > %t/tertiary-overlay.yaml
+// RUN: sed -e "s|INPUT_DIR|%/S/Inputs|g" -e "s|OUT_DIR|%/t|g" %S/Inputs/vfs/quaternary-vfsoverlay.yaml > %t/quaternary-vfsoverlay.yaml
 
 // RUN: not %target-swift-frontend -vfsoverlay %/t/overlay.yaml -typecheck %s %/t/mapped-file.swift -serialize-diagnostics-path %/t/basic.dia 2>&1 | %FileCheck -check-prefix=BASIC_MAPPING_ERROR %s
 // RUN: c-index-test -read-diagnostics %/t/basic.dia 2>&1 | %FileCheck -check-prefix=BASIC_MAPPING_ERROR %s
@@ -20,10 +21,11 @@
 // RUN: %target-swift-frontend(mock-sdk: %clang-importer-sdk) -I %/t -DTEST_VFS_CLANG_IMPORTER -Xcc -ivfsoverlay -Xcc %/t/overlay.yaml -typecheck %/s
 
 // If we see -ivfsoverlay and -vfsoverlay, we'll clobber Clang's VFS with our own.
-// RUN: %target-swift-frontend(mock-sdk: %clang-importer-sdk) -I %/t -DTEST_VFS_CLANG_IMPORTER -vfsoverlay %/t/overlay.yaml -Xcc -ivfsoverlay -Xcc %/t/overlay.yaml -typecheck %/s  2>&1 | %FileCheck -check-prefix=WARN_VFS_CLOBBERED %s
-
-// WARN_VFS_CLOBBERED: warning: ignoring '-ivfsoverlay' options provided to '-Xcc' in favor of '-vfsoverlay'
+// RUN: %target-swift-frontend(mock-sdk: %clang-importer-sdk) -I %/t -DTEST_VFS_CLANG_IMPORTER -DTEST_VFS_CLANG_IMPORTER_MERGE -vfsoverlay %/t/overlay.yaml -Xcc -ivfsoverlay -Xcc %/t/quaternary-vfsoverlay.yaml -typecheck %/s  2>&1
 
 #if TEST_VFS_CLANG_IMPORTER
 import VFSMappedModule
+#if TEST_VFS_CLANG_IMPORTER_MERGE
+import YetAnotherVFSMappedModule
+#endif
 #endif

unittests/FrontendTool/ModuleLoadingTests.cpp

@@ -87,8 +87,10 @@ class ParseableInterfaceModuleLoaderTest : public testing::Test {
     SourceManager sourceMgr;
 
     // Create a file system that tracks how many times a file has been opened.
-    llvm::IntrusiveRefCntPtr<OpenTrackingFileSystem> fs(
-      new OpenTrackingFileSystem(sourceMgr.getFileSystem()));
+    llvm::IntrusiveRefCntPtr<OpenTrackingFileSystem> baseFS(
+        new OpenTrackingFileSystem(sourceMgr.getFileSystem()));
+    auto fs = llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>(
+        new llvm::vfs::OverlayFileSystem(baseFS));
 
     sourceMgr.setFileSystem(fs);
     PrintingDiagnosticConsumer printingConsumer;
@@ -126,7 +128,7 @@ class ParseableInterfaceModuleLoaderTest : public testing::Test {
     ASSERT_TRUE(fs->exists(cachedModulePath));
 
     // Assert that we've only opened this file once, to write it.
-    ASSERT_EQ((unsigned)1, fs->numberOfOpens(cachedModulePath));
+    ASSERT_EQ((unsigned)1, baseFS->numberOfOpens(cachedModulePath));
 
     auto bufOrErr = fs->getBufferForFile(cachedModulePath);
     ASSERT_TRUE(bufOrErr);