Merge pull request #4267 from DavidGoldman/stable/20211026

Support hermetic index store data via path remappings #4207

Author: bnbarham

clang/include/clang/Basic/PathRemapper.h

@@ -0,0 +1,74 @@
+//===--- PathRemapper.h - Remap filepath prefixes ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines a data structure that stores a string-to-string
+//  mapping used to transform file paths based on a prefix mapping. It
+//  is optimized for the common case of having 0, 1, or 2 mappings.
+//
+//  Remappings are stored such that they are applied in the order they
+//  are passed on the command line, with the first one winning - a path will
+//  only be remapped by a single mapping, if any, not multiple. The ordering
+//  would only matter if one source mapping was a prefix of another.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_BASIC_PATHREMAPPER_H
+#define LLVM_CLANG_BASIC_PATHREMAPPER_H
+
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/Path.h"
+
+#include <string>
+#include <utility>
+
+namespace clang {
+
+class PathRemapper {
+  SmallVector<std::pair<std::string, std::string>, 2> PathMappings;
+public:
+  /// Adds a mapping such that any paths starting with `FromPrefix` have that
+  /// portion replaced with `ToPrefix`.
+  void addMapping(StringRef FromPrefix, StringRef ToPrefix) {
+    PathMappings.emplace_back(FromPrefix.str(), ToPrefix.str());
+  }
+
+  /// Returns a remapped `Path` if it starts with a prefix in the remapper;
+  /// otherwise the original path is returned.
+  std::string remapPath(StringRef Path) const {
+    for (const auto &Mapping : PathMappings)
+      if (Path.startswith(Mapping.first))
+        return (Twine(Mapping.second) +
+                Path.substr(Mapping.first.size())).str();
+    return Path.str();
+  }
+
+  /// Remaps `PathBuf` if it starts with a prefix in the remapper. Avoids any
+  /// heap allocations if the path is not remapped.
+  void remapPath(SmallVectorImpl<char> &PathBuf) const {
+    for (const auto &E : PathMappings)
+      if (llvm::sys::path::replace_path_prefix(PathBuf, E.first, E.second))
+        break;
+  }
+
+  /// Return true if there are no path remappings (meaning remapPath will always
+  /// return the path given).
+  bool empty() const {
+    return PathMappings.empty();
+  }
+
+  ArrayRef<std::pair<std::string, std::string>> getMappings() const {
+    return PathMappings;
+  }
+};
+
+} // end namespace clang
+
+#endif

clang/include/clang/Index/IndexDataStore.h

@@ -10,9 +10,10 @@
 #define LLVM_CLANG_INDEX_INDEXDATASTORE_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/PathRemapper.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Chrono.h"
 #include <functional>
 #include <memory>
@@ -25,10 +26,12 @@ class IndexDataStore {
 public:
   ~IndexDataStore();
 
-  static std::unique_ptr<IndexDataStore>
-    create(StringRef IndexStorePath, std::string &Error);
+  static std::unique_ptr<IndexDataStore> create(StringRef IndexStorePath,
+                                                const PathRemapper &Remapper,
+                                                std::string &Error);
 
   StringRef getFilePath() const;
+  const PathRemapper &getPathRemapper() const;
   bool foreachUnitName(bool sorted,
                        llvm::function_ref<bool(StringRef unitName)> receiver);
 

clang/include/clang/Index/IndexUnitReader.h

@@ -10,6 +10,7 @@
 #define LLVM_CLANG_INDEX_INDEXUNITREADER_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/PathRemapper.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Chrono.h"
@@ -29,9 +30,10 @@ class IndexUnitReader {
 
   static std::unique_ptr<IndexUnitReader>
     createWithUnitFilename(StringRef UnitFilename, StringRef StorePath,
-                           std::string &Error);
+                           const PathRemapper &Remapper, std::string &Error);
   static std::unique_ptr<IndexUnitReader>
-    createWithFilePath(StringRef FilePath, std::string &Error);
+    createWithFilePath(StringRef FilePath, const PathRemapper &Remapper,
+                       std::string &Error);
 
   static Optional<llvm::sys::TimePoint<>>
     getModificationTimeForUnit(StringRef UnitFilename, StringRef StorePath,

clang/include/clang/Index/IndexUnitWriter.h

@@ -22,6 +22,7 @@ namespace llvm {
 namespace clang {
   class FileEntry;
   class FileManager;
+  class PathRemapper;
 
 namespace index {
 
@@ -57,6 +58,7 @@ class IndexUnitWriter {
   std::string TargetTriple;
   std::string WorkDir;
   std::string SysrootPath;
+  const PathRemapper &Remapper;
   std::function<writer::ModuleInfo(writer::OpaqueModule,
                             SmallVectorImpl<char> &Scratch)> GetInfoForModuleFn;
   struct FileInclude {
@@ -87,6 +89,8 @@ class IndexUnitWriter {
   /// \param MainFile the main file for a compiled source file. This should be
   /// null for PCH and module units.
   /// \param IsSystem true for system module units, false otherwise.
+  /// \param Remapper Remapper to use to standardize file paths to make them
+  /// hermetic/reproducible. This applies to all paths emitted in the unit file.
   IndexUnitWriter(FileManager &FileMgr,
                   StringRef StorePath,
                   StringRef ProviderIdentifier, StringRef ProviderVersion,
@@ -98,6 +102,7 @@ class IndexUnitWriter {
                   bool IsDebugCompilation,
                   StringRef TargetTriple,
                   StringRef SysrootPath,
+                  const PathRemapper &Remapper,
                   writer::ModuleInfoWriterCallback GetInfoForModule);
   ~IndexUnitWriter();
 
@@ -120,7 +125,8 @@ class IndexUnitWriter {
   Optional<bool> isUnitUpToDateForOutputFile(StringRef FilePath,
                                              Optional<StringRef> TimeCompareFilePath,
                                              std::string &Error);
-  static void getUnitNameForAbsoluteOutputFile(StringRef FilePath, SmallVectorImpl<char> &Str);
+  static void getUnitNameForAbsoluteOutputFile(StringRef FilePath, SmallVectorImpl<char> &Str,
+                                               const PathRemapper &Remapper);
   static bool initIndexDirectory(StringRef StorePath, std::string &Error);
 
 private:

clang/include/indexstore/IndexStoreCXX.h

@@ -14,6 +14,7 @@
 #define LLVM_CLANG_INDEXSTORE_INDEXSTORECXX_H
 
 #include "indexstore/indexstore.h"
+#include "clang/Basic/PathRemapper.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
@@ -101,10 +102,16 @@ class IndexStore {
   friend class IndexUnitReader;
 
 public:
-  IndexStore(StringRef path, std::string &error) {
+  IndexStore(StringRef path, const clang::PathRemapper &remapper,
+             std::string &error) {
     llvm::SmallString<64> buf = path;
     indexstore_error_t c_err = nullptr;
-    obj = indexstore_store_create(buf.c_str(), &c_err);
+    indexstore_creation_options_t options = indexstore_creation_options_create();
+    for (const auto &Mapping : remapper.getMappings())
+      indexstore_creation_options_add_prefix_mapping(options, Mapping.first.c_str(), Mapping.second.c_str());
+
+    obj = indexstore_store_create_with_options(buf.c_str(), options, &c_err);
+    indexstore_creation_options_dispose(options);
     if (c_err) {
       error = indexstore_error_get_description(c_err);
       indexstore_error_dispose(c_err);
@@ -119,8 +126,9 @@ class IndexStore {
     indexstore_store_dispose(obj);
   }
 
-  static IndexStoreRef create(StringRef path, std::string &error) {
-    auto storeRef = std::make_shared<IndexStore>(path, error);
+  static IndexStoreRef create(StringRef path, clang::PathRemapper remapper,
+                              std::string &error) {
+    auto storeRef = std::make_shared<IndexStore>(path, remapper, error);
     if (storeRef->isInvalid())
       return nullptr;
     return storeRef;

clang/include/indexstore/indexstore.h

@@ -25,7 +25,7 @@
  * INDEXSTORE_VERSION_MAJOR is intended for "major" source/ABI breaking changes.
  */
 #define INDEXSTORE_VERSION_MAJOR 0
-#define INDEXSTORE_VERSION_MINOR 12
+#define INDEXSTORE_VERSION_MINOR 13
 
 #define INDEXSTORE_VERSION_ENCODE(major, minor) ( \
       ((major) * 10000)                           \
@@ -129,10 +129,32 @@ INDEXSTORE_PUBLIC unsigned
 indexstore_format_version(void);
 
 typedef void *indexstore_t;
+typedef void *indexstore_creation_options_t;
+
+INDEXSTORE_PUBLIC indexstore_creation_options_t
+indexstore_creation_options_create(void);
+
+INDEXSTORE_PUBLIC void
+indexstore_creation_options_dispose(indexstore_creation_options_t);
+
+/// Adds a remapping from \c path_prefix to \c remapped_path_prefix.
+///
+/// This should be used to convert hermetic or remote paths embedded in the index data to the
+/// equivalent paths on the local machine.
+INDEXSTORE_PUBLIC void
+indexstore_creation_options_add_prefix_mapping(indexstore_creation_options_t options,
+                                               const char *path_prefix,
+                                               const char *remapped_path_prefix);
 
 INDEXSTORE_PUBLIC indexstore_t
 indexstore_store_create(const char *store_path, indexstore_error_t *error);
 
+
+/// Open the indexstore at the specified path using the specified options, which may be NULL.
+INDEXSTORE_PUBLIC indexstore_t
+indexstore_store_create_with_options(const char *store_path, indexstore_creation_options_t options,
+                                     indexstore_error_t *error);
+
 INDEXSTORE_PUBLIC void
 indexstore_store_dispose(indexstore_t);
 

clang/lib/Index/IndexUnitReader.cpp

@@ -43,14 +43,15 @@ class IndexUnitReaderImpl {
   bool IsSystemUnit;
   bool IsModuleUnit;
   bool IsDebugCompilation;
-  StringRef WorkingDir;
-  StringRef OutputFile;
-  StringRef SysrootPath;
+  std::string WorkingDir;
+  std::string OutputFile;
+  std::string SysrootPath;
   StringRef ModuleName;
   SmallString<128> MainFilePath;
   StringRef Target;
   std::vector<FileBitPath> Paths;
   StringRef PathsBuffer;
+  const PathRemapper &Remapper;
 
   struct ModuleInfo {
     unsigned NameOffset;
@@ -59,6 +60,7 @@ class IndexUnitReaderImpl {
   std::vector<ModuleInfo> Modules;
   StringRef ModuleNamesBuffer;
 
+  IndexUnitReaderImpl(const PathRemapper &remapper) : Remapper(remapper) {}
   bool init(std::unique_ptr<MemoryBuffer> Buf, sys::TimePoint<> ModTime,
             std::string &Error);
 
@@ -88,6 +90,10 @@ class IndexUnitReaderImpl {
     return PathsBuffer.substr(Offset, Size);
   }
 
+  std::string getAndRemapPathFromBuffer(size_t Offset, size_t Size) {
+    return Remapper.remapPath(getPathFromBuffer(Offset, Size));
+  }
+
   void constructFilePath(SmallVectorImpl<char> &Path, int PathIndex);
 
   StringRef getModuleName(int ModuleIndex);
@@ -205,9 +211,9 @@ class IndexUnitBitstreamVisitor : public BitstreamVisitor<IndexUnitBitstreamVisi
         break;
       case UNIT_PATH_BUFFER:
         Reader.PathsBuffer = Blob;
-        Reader.WorkingDir = Reader.getPathFromBuffer(WorkDirOffset, WorkDirSize);
-        Reader.OutputFile = Reader.getPathFromBuffer(OutputFileOffset, OutputFileSize);
-        Reader.SysrootPath = Reader.getPathFromBuffer(SysrootOffset, SysrootSize);
+        Reader.WorkingDir = Reader.getAndRemapPathFromBuffer(WorkDirOffset, WorkDirSize);
+        Reader.OutputFile = Reader.getAndRemapPathFromBuffer(OutputFileOffset, OutputFileSize);
+        Reader.SysrootPath = Reader.getAndRemapPathFromBuffer(SysrootOffset, SysrootSize);
 
         // now we can populate the main file's path
         Reader.constructFilePath(Reader.MainFilePath, MainPathIndex);
@@ -270,7 +276,8 @@ class IndexUnitBlockBitstreamVisitor : public BitstreamVisitor<IndexUnitBlockBit
 } // anonymous namespace
 
 bool IndexUnitReaderImpl::init(std::unique_ptr<MemoryBuffer> Buf,
-                               sys::TimePoint<> ModTime, std::string &Error) {
+                               sys::TimePoint<> ModTime,
+                               std::string &Error) {
   this->ModTime = ModTime;
   this->MemBuf = std::move(Buf);
   llvm::BitstreamCursor Stream(*MemBuf);
@@ -374,6 +381,8 @@ void IndexUnitReaderImpl::constructFilePath(SmallVectorImpl<char> &PathBuf,
   sys::path::append(PathBuf,
                     getPathFromBuffer(Path.Dir.Offset, Path.Dir.Size),
                     getPathFromBuffer(Path.Filename.Offset, Path.Filename.Size));
+  if (Path.PrefixKind == UNIT_PATH_PREFIX_NONE && !Remapper.empty())
+    Remapper.remapPath(PathBuf);
 }
 
 StringRef IndexUnitReaderImpl::getModuleName(int ModuleIndex) {
@@ -391,15 +400,18 @@ StringRef IndexUnitReaderImpl::getModuleName(int ModuleIndex) {
 std::unique_ptr<IndexUnitReader>
 IndexUnitReader::createWithUnitFilename(StringRef UnitFilename,
                                         StringRef StorePath,
+                                        const PathRemapper &Remapper,
                                         std::string &Error) {
   SmallString<128> PathBuf = StorePath;
   appendUnitSubDir(PathBuf);
   sys::path::append(PathBuf, UnitFilename);
-  return createWithFilePath(PathBuf.str(), Error);
+  return createWithFilePath(PathBuf.str(), Remapper, Error);
 }
 
 std::unique_ptr<IndexUnitReader>
-IndexUnitReader::createWithFilePath(StringRef FilePath, std::string &Error) {
+IndexUnitReader::createWithFilePath(StringRef FilePath,
+                                    const PathRemapper &Remapper,
+                                    std::string &Error) {
   int FD;
   std::error_code EC = sys::fs::openFileForRead(FilePath, FD);
   if (EC) {
@@ -433,7 +445,7 @@ IndexUnitReader::createWithFilePath(StringRef FilePath, std::string &Error) {
     return nullptr;
   }
 
-  std::unique_ptr<IndexUnitReaderImpl> Impl(new IndexUnitReaderImpl());
+  auto Impl = std::make_unique<IndexUnitReaderImpl>(Remapper);
   bool Err = Impl->init(std::move(*ErrOrBuf), FileStat.getLastModificationTime(),
                         Error);
   if (Err)