Skip to content

Commit aa5fcf6

Browse files
benlangmuirbenlangmuir
authored
Merge pull request #10406 from benlangmuir/validate-ac
[llvm][cas] Extend on-disk CAS validation to ActionCache
2 parents 8cb7660 + 2966de4 commit aa5fcf6

File tree

11 files changed

+106
-3
lines changed

11 files changed

+106
-3
lines changed

‎llvm/include/llvm/CAS/ActionCache.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,9 @@ class ActionCache {
114114
Globally, std::move(Callback), CancelObj);
115115
}
116116

117+
/// Validate the ActionCache contents.
118+
virtual Error validate() const = 0;
119+
117120
virtual ~ActionCache() = default;
118121

119122
protected:

‎llvm/include/llvm/CAS/OnDiskKeyValueDB.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ class OnDiskKeyValueDB {
6060
StringRef ValueName, size_t ValueSize,
6161
std::shared_ptr<OnDiskCASLogger> Logger = nullptr);
6262

63+
using CheckValueT = function_ref<Error(FileOffset Offset, ArrayRef<char>)>;
64+
Error validate(CheckValueT CheckValue) const;
65+
6366
private:
6467
OnDiskKeyValueDB(size_t ValueSize, OnDiskHashMappedTrie Cache)
6568
: ValueSize(ValueSize), Cache(std::move(Cache)) {}

‎llvm/include/llvm/CAS/UnifiedOnDiskCache.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ class UnifiedOnDiskCache {
124124

125125
~UnifiedOnDiskCache();
126126

127+
Error validateActionCache();
128+
127129
private:
128130
UnifiedOnDiskCache();
129131

‎llvm/lib/CAS/ActionCaches.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ class InMemoryActionCache final : public ActionCache {
5252
Expected<std::optional<CASID>> getImpl(ArrayRef<uint8_t> ActionKey,
5353
bool Globally) const final;
5454

55+
Error validate() const final {
56+
return createStringError("InMemoryActionCache doesn't support validate()");
57+
}
58+
5559
private:
5660
using DataT = CacheEntry<sizeof(HashType)>;
5761
using InMemoryCacheT = ThreadSafeHashMappedTrie<DataT, sizeof(HashType)>;
@@ -68,6 +72,8 @@ class OnDiskActionCache final : public ActionCache {
6872

6973
static Expected<std::unique_ptr<OnDiskActionCache>> create(StringRef Path);
7074

75+
Error validate() const final;
76+
7177
private:
7278
static StringRef getHashName() { return "BLAKE3"; }
7379

@@ -86,6 +92,8 @@ class UnifiedOnDiskActionCache final : public ActionCache {
8692

8793
UnifiedOnDiskActionCache(std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB);
8894

95+
Error validate() const final;
96+
8997
private:
9098
std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB;
9199
};
@@ -198,6 +206,12 @@ Error OnDiskActionCache::putImpl(ArrayRef<uint8_t> Key, const CASID &Result,
198206
ArrayRef((const uint8_t *)Observed.data(), Observed.size()));
199207
}
200208

209+
Error OnDiskActionCache::validate() const {
210+
// FIXME: without the matching CAS there is nothing we can check about the
211+
// cached values. The hash size is already validated by the DB validator.
212+
return DB->validate(nullptr);
213+
}
214+
201215
UnifiedOnDiskActionCache::UnifiedOnDiskActionCache(
202216
std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB)
203217
: ActionCache(builtin::BuiltinCASContext::getDefaultContext()),
@@ -233,6 +247,10 @@ Error UnifiedOnDiskActionCache::putImpl(ArrayRef<uint8_t> Key,
233247
UniDB->getGraphDB().getDigest(*Observed));
234248
}
235249

250+
Error UnifiedOnDiskActionCache::validate() const {
251+
return UniDB->validateActionCache();
252+
}
253+
236254
Expected<std::unique_ptr<ActionCache>>
237255
cas::createOnDiskActionCache(StringRef Path) {
238256
#if LLVM_ENABLE_ONDISK_CAS

‎llvm/lib/CAS/OnDiskKeyValueDB.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,3 +81,25 @@ OnDiskKeyValueDB::open(StringRef Path, StringRef HashName, unsigned KeySize,
8181
return std::unique_ptr<OnDiskKeyValueDB>(
8282
new OnDiskKeyValueDB(ValueSize, std::move(*ActionCache)));
8383
}
84+
85+
Error OnDiskKeyValueDB::validate(CheckValueT CheckValue) const {
86+
return Cache.validate(
87+
[&](FileOffset Offset,
88+
OnDiskHashMappedTrie::ConstValueProxy Record) -> Error {
89+
auto formatError = [&](Twine Msg) {
90+
return createStringError(
91+
llvm::errc::illegal_byte_sequence,
92+
"bad cache value at 0x" +
93+
utohexstr((unsigned)Offset.get(), /*LowerCase=*/true) + ": " +
94+
Msg.str());
95+
};
96+
97+
if (Record.Data.size() != ValueSize)
98+
return formatError("wrong cache value size");
99+
if (!isAligned(Align(8), Record.Data.size()))
100+
return formatError("wrong cache value alignment");
101+
if (CheckValue)
102+
return CheckValue(Offset, Record.Data);
103+
return Error::success();
104+
});
105+
}

‎llvm/lib/CAS/PluginAPI.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,8 @@ struct llcas_functions_t {
108108
bool globally, void *ctx_cb,
109109
llcas_actioncache_put_cb,
110110
llcas_cancellable_t *);
111+
112+
bool (*actioncache_validate)(llcas_cas_t, char **error);
111113
};
112114

113115
#endif // LLVM_LIB_CAS_PLUGINAPI_H

‎llvm/lib/CAS/PluginAPI_functions.def

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ CASPLUGINAPI_FUNCTION(actioncache_get_for_digest, true)
77
CASPLUGINAPI_FUNCTION(actioncache_get_for_digest_async, true)
88
CASPLUGINAPI_FUNCTION(actioncache_put_for_digest, true)
99
CASPLUGINAPI_FUNCTION(actioncache_put_for_digest_async, true)
10+
CASPLUGINAPI_FUNCTION(actioncache_validate, false)
1011
CASPLUGINAPI_FUNCTION(cancellable_cancel, false)
1112
CASPLUGINAPI_FUNCTION(cancellable_dispose, false)
1213
CASPLUGINAPI_FUNCTION(cas_contains_object, true)

‎llvm/lib/CAS/PluginCAS.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,8 @@ class PluginActionCache : public ActionCache {
461461

462462
PluginActionCache(std::shared_ptr<PluginCASContext>);
463463

464+
Error validate() const final;
465+
464466
private:
465467
std::shared_ptr<PluginCASContext> Ctx;
466468
};
@@ -596,6 +598,16 @@ void PluginActionCache::putImplAsync(ArrayRef<uint8_t> ResolvedKey,
596598
PluginActionCache::PluginActionCache(std::shared_ptr<PluginCASContext> CASCtx)
597599
: ActionCache(*CASCtx), Ctx(std::move(CASCtx)) {}
598600

601+
Error PluginActionCache::validate() const {
602+
if (Ctx->Functions.actioncache_validate) {
603+
char *c_err = nullptr;
604+
if (Ctx->Functions.actioncache_validate(Ctx->c_cas, &c_err))
605+
return Ctx->errorAndDispose(c_err);
606+
return Error::success();
607+
}
608+
return createStringError("plugin action cache doesn't support validation");
609+
}
610+
599611
//===----------------------------------------------------------------------===//
600612
// createPluginCASDatabases API
601613
//===----------------------------------------------------------------------===//

‎llvm/lib/CAS/UnifiedOnDiskCache.cpp

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
#include "llvm/CAS/UnifiedOnDiskCache.h"
5353
#include "OnDiskCommon.h"
5454
#include "llvm/ADT/ScopeExit.h"
55+
#include "llvm/ADT/StringExtras.h"
5556
#include "llvm/CAS/OnDiskCASLogger.h"
5657
#include "llvm/CAS/OnDiskKeyValueDB.h"
5758
#include "llvm/Support/Compiler.h"
@@ -123,6 +124,30 @@ UnifiedOnDiskCache::faultInFromUpstreamKV(ArrayRef<uint8_t> Key) {
123124
return KVPut(Key, *PrimaryID);
124125
}
125126

127+
Error UnifiedOnDiskCache::validateActionCache() {
128+
auto ValidateRef = [&](FileOffset Offset, ArrayRef<char> Value) -> Error {
129+
assert(Value.size() == sizeof(uint64_t) && "should be validated already");
130+
auto ID = ObjectID::fromOpaqueData(support::endian::read64le(Value.data()));
131+
auto formatError = [&](Twine Msg) {
132+
return createStringError(
133+
llvm::errc::illegal_byte_sequence,
134+
"bad record at 0x" +
135+
utohexstr((unsigned)Offset.get(), /*LowerCase=*/true) + ": " +
136+
Msg.str());
137+
};
138+
if (ID.getOpaqueData() == 0)
139+
return formatError("zero is not a valid ref");
140+
if (!PrimaryGraphDB->containsObject(ID))
141+
return formatError("cas does not contain ref");
142+
return Error::success();
143+
};
144+
if (Error E = PrimaryKVDB->validate(ValidateRef))
145+
return E;
146+
if (UpstreamKVDB)
147+
return UpstreamKVDB->validate(ValidateRef);
148+
return Error::success();
149+
}
150+
126151
/// \returns all the 'v<version>.<x>' names of sub-directories, sorted with
127152
/// ascending order of the integer after the dot.
128153
static Error getAllDBDirs(StringRef Path,

‎llvm/test/tools/llvm-cas/validation.test

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,17 @@ RUN: rm %t/cas/v1.1/v8.data
99
RUN: not llvm-cas --cas %t/cas --validate
1010
RUN: not llvm-cas --cas %t/cas --validate --check-hash
1111

12+
RUN: mkdir %t/ac
13+
14+
RUN: llvm-cas --cas %t/ac --make-blob \
15+
RUN: --data /dev/null > %t/empty.casid
16+
RUN: echo "abc" | \
17+
RUN: llvm-cas --cas %t/ac --make-blob \
18+
RUN: --data - >%t/abc.casid
19+
20+
RUN: llvm-cas --cas %t/ac --put-cache-key @%t/abc.casid @%t/empty.casid
21+
RUN: llvm-cas --cas %t/ac --validate
22+
# Note: records are 40 bytes (32 hash bytes + 8 byte value), so trim the last
23+
# allocated record, leaving it invalid.
24+
RUN: truncate -s -40 %t/ac/v1.1/v3.actions
25+
RUN: not llvm-cas --cas %t/ac --validate

‎llvm/tools/llvm-cas/llvm-cas.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
6464
ArrayRef<std::string> Objects);
6565
static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID);
6666
static int validateObject(ObjectStore &CAS, const CASID &ID);
67-
static int validate(ObjectStore &CAS, bool CheckHash);
67+
static int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash);
6868
static int ingestCasIDFile(cas::ObjectStore &CAS, ArrayRef<std::string> CASIDs);
6969
static int checkLockFiles(StringRef CASPath);
7070

@@ -184,7 +184,7 @@ int main(int Argc, char **Argv) {
184184
return dump(*CAS);
185185

186186
if (Command == Validate)
187-
return validate(*CAS, CheckHash);
187+
return validate(*CAS, *AC, CheckHash);
188188

189189
if (Command == MakeBlob)
190190
return makeBlob(*CAS, DataPath);
@@ -722,9 +722,10 @@ int validateObject(ObjectStore &CAS, const CASID &ID) {
722722
return 0;
723723
}
724724

725-
int validate(ObjectStore &CAS, bool CheckHash) {
725+
int validate(ObjectStore &CAS, ActionCache &AC, bool CheckHash) {
726726
ExitOnError ExitOnErr("llvm-cas: validate: ");
727727
ExitOnErr(CAS.validate(CheckHash));
728+
ExitOnErr(AC.validate());
728729
outs() << "validated successfully\n";
729730
return 0;
730731
}

0 commit comments

Comments
 (0)