|
| 1 | +//===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===// |
| 2 | +// |
| 3 | +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | +// See https://llvm.org/LICENSE.txt for license information. |
| 5 | +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | +// |
| 7 | +//===----------------------------------------------------------------------===// |
| 8 | + |
| 9 | +#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H |
| 10 | +#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H |
| 11 | + |
| 12 | +#include "clang/Basic/LLVM.h" |
| 13 | +#include "llvm/ADT/StringMap.h" |
| 14 | +#include "llvm/ADT/StringSet.h" |
| 15 | +#include "llvm/Support/Allocator.h" |
| 16 | +#include "llvm/Support/ErrorOr.h" |
| 17 | +#include "llvm/Support/VirtualFileSystem.h" |
| 18 | +#include <mutex> |
| 19 | + |
| 20 | +namespace clang { |
| 21 | +namespace tooling { |
| 22 | +namespace dependencies { |
| 23 | + |
| 24 | +/// An in-memory representation of a file system entity that is of interest to |
| 25 | +/// the dependency scanning filesystem. |
| 26 | +/// |
| 27 | +/// It represents one of the following: |
| 28 | +/// - an opened source file with minimized contents and a stat value. |
| 29 | +/// - an opened source file with original contents and a stat value. |
| 30 | +/// - a directory entry with its stat value. |
| 31 | +/// - an error value to represent a file system error. |
| 32 | +/// - a placeholder with an invalid stat indicating a not yet initialized entry. |
| 33 | +class CachedFileSystemEntry { |
| 34 | +public: |
| 35 | + /// Default constructor creates an entry with an invalid stat. |
| 36 | + CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {} |
| 37 | + |
| 38 | + CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {} |
| 39 | + |
| 40 | + /// Create an entry that represents an opened source file with minimized or |
| 41 | + /// original contents. |
| 42 | + /// |
| 43 | + /// The filesystem opens the file even for `stat` calls open to avoid the |
| 44 | + /// issues with stat + open of minimized files that might lead to a |
| 45 | + /// mismatching size of the file. If file is not minimized, the full file is |
| 46 | + /// read and copied into memory to ensure that it's not memory mapped to avoid |
| 47 | + /// running out of file descriptors. |
| 48 | + static CachedFileSystemEntry createFileEntry(StringRef Filename, |
| 49 | + llvm::vfs::FileSystem &FS, |
| 50 | + bool Minimize = true); |
| 51 | + |
| 52 | + /// Create an entry that represents a directory on the filesystem. |
| 53 | + static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat); |
| 54 | + |
| 55 | + /// \returns True if the entry is valid. |
| 56 | + bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); } |
| 57 | + |
| 58 | + /// \returns The error or the file's contents. |
| 59 | + llvm::ErrorOr<StringRef> getContents() const { |
| 60 | + if (!MaybeStat) |
| 61 | + return MaybeStat.getError(); |
| 62 | + assert(!MaybeStat->isDirectory() && "not a file"); |
| 63 | + assert(isValid() && "not initialized"); |
| 64 | + return StringRef(Contents); |
| 65 | + } |
| 66 | + |
| 67 | + /// \returns The error or the status of the entry. |
| 68 | + llvm::ErrorOr<llvm::vfs::Status> getStatus() const { |
| 69 | + assert(isValid() && "not initialized"); |
| 70 | + return MaybeStat; |
| 71 | + } |
| 72 | + |
| 73 | + /// \returns the name of the file. |
| 74 | + StringRef getName() const { |
| 75 | + assert(isValid() && "not initialized"); |
| 76 | + return MaybeStat->getName(); |
| 77 | + } |
| 78 | + |
| 79 | + CachedFileSystemEntry(CachedFileSystemEntry &&) = default; |
| 80 | + CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default; |
| 81 | + |
| 82 | + CachedFileSystemEntry(const CachedFileSystemEntry &) = delete; |
| 83 | + CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete; |
| 84 | + |
| 85 | +private: |
| 86 | + llvm::ErrorOr<llvm::vfs::Status> MaybeStat; |
| 87 | + // Store the contents in a small string to allow a |
| 88 | + // move from the small string for the minimized contents. |
| 89 | + // Note: small size of 1 allows us to store an empty string with an implicit |
| 90 | + // null terminator without any allocations. |
| 91 | + llvm::SmallString<1> Contents; |
| 92 | +}; |
| 93 | + |
| 94 | +/// This class is a shared cache, that caches the 'stat' and 'open' calls to the |
| 95 | +/// underlying real file system. |
| 96 | +/// |
| 97 | +/// It is sharded based on the hash of the key to reduce the lock contention for |
| 98 | +/// the worker threads. |
| 99 | +class DependencyScanningFilesystemSharedCache { |
| 100 | +public: |
| 101 | + struct SharedFileSystemEntry { |
| 102 | + std::mutex ValueLock; |
| 103 | + CachedFileSystemEntry Value; |
| 104 | + }; |
| 105 | + |
| 106 | + DependencyScanningFilesystemSharedCache(); |
| 107 | + |
| 108 | + /// Returns a cache entry for the corresponding key. |
| 109 | + /// |
| 110 | + /// A new cache entry is created if the key is not in the cache. This is a |
| 111 | + /// thread safe call. |
| 112 | + SharedFileSystemEntry &get(StringRef Key); |
| 113 | + |
| 114 | +private: |
| 115 | + struct CacheShard { |
| 116 | + std::mutex CacheLock; |
| 117 | + llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache; |
| 118 | + }; |
| 119 | + std::unique_ptr<CacheShard[]> CacheShards; |
| 120 | + unsigned NumShards; |
| 121 | +}; |
| 122 | + |
| 123 | +/// A virtual file system optimized for the dependency discovery. |
| 124 | +/// |
| 125 | +/// It is primarily designed to work with source files whose contents was was |
| 126 | +/// preprocessed to remove any tokens that are unlikely to affect the dependency |
| 127 | +/// computation. |
| 128 | +/// |
| 129 | +/// This is not a thread safe VFS. A single instance is meant to be used only in |
| 130 | +/// one thread. Multiple instances are allowed to service multiple threads |
| 131 | +/// running in parallel. |
| 132 | +class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem { |
| 133 | +public: |
| 134 | + DependencyScanningWorkerFilesystem( |
| 135 | + DependencyScanningFilesystemSharedCache &SharedCache, |
| 136 | + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) |
| 137 | + : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache) {} |
| 138 | + |
| 139 | + llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override; |
| 140 | + llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
| 141 | + openFileForRead(const Twine &Path) override; |
| 142 | + |
| 143 | + /// The set of files that should not be minimized. |
| 144 | + llvm::StringSet<> IgnoredFiles; |
| 145 | + |
| 146 | +private: |
| 147 | + void setCachedEntry(StringRef Filename, const CachedFileSystemEntry *Entry) { |
| 148 | + bool IsInserted = Cache.try_emplace(Filename, Entry).second; |
| 149 | + (void)IsInserted; |
| 150 | + assert(IsInserted && "local cache is updated more than once"); |
| 151 | + } |
| 152 | + |
| 153 | + const CachedFileSystemEntry *getCachedEntry(StringRef Filename) { |
| 154 | + auto It = Cache.find(Filename); |
| 155 | + return It == Cache.end() ? nullptr : It->getValue(); |
| 156 | + } |
| 157 | + |
| 158 | + DependencyScanningFilesystemSharedCache &SharedCache; |
| 159 | + /// The local cache is used by the worker thread to cache file system queries |
| 160 | + /// locally instead of querying the global cache every time. |
| 161 | + llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache; |
| 162 | +}; |
| 163 | + |
| 164 | +} // end namespace dependencies |
| 165 | +} // end namespace tooling |
| 166 | +} // end namespace clang |
| 167 | + |
| 168 | +#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H |
0 commit comments