Skip to content

Commit 4a16fe1

Browse files
noajshunoajshu
committed
[llvm] [Debuginfo] Debuginfod client library.
This adds a Debuginfod client library which queries servers specified by the `DEBUGINFOD_URLS` environment variable for the debuginfo, executable, or a specified source file associated with a given build id. Reviewed By: dblaikie Differential Revision: https://reviews.llvm.org/D112758
1 parent 6c75ab5 commit 4a16fe1

File tree

8 files changed

+316
-3
lines changed

8 files changed

+316
-3
lines changed

‎llvm/include/llvm/Debuginfod/Debuginfod.h

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
//===-- llvm/Debuginfod/Debuginfod.h - Debuginfod client --------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
///
9+
/// \file
10+
/// This file contains the declarations of getCachedOrDownloadArtifact and
11+
/// several convenience functions for specific artifact types:
12+
/// getCachedOrDownloadSource, getCachedOrDownloadExecutable, and
13+
/// getCachedOrDownloadDebuginfo. This file also declares
14+
/// getDefaultDebuginfodUrls and getDefaultDebuginfodCacheDirectory.
15+
///
16+
///
17+
//===----------------------------------------------------------------------===//
18+
19+
#ifndef LLVM_DEBUGINFOD_DEBUGINFOD_H
20+
#define LLVM_DEBUGINFOD_DEBUGINFOD_H
21+
22+
#include "llvm/ADT/StringRef.h"
23+
#include "llvm/Support/Error.h"
24+
#include "llvm/Support/MemoryBuffer.h"
25+
26+
namespace llvm {
27+
28+
typedef ArrayRef<uint8_t> BuildIDRef;
29+
30+
typedef SmallVector<uint8_t, 10> BuildID;
31+
32+
/// Finds default array of Debuginfod server URLs by checking DEBUGINFOD_URLS
33+
/// environment variable.
34+
Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls();
35+
36+
/// Finds a default local file caching directory for the debuginfod client,
37+
/// first checking DEBUGINFOD_CACHE_PATH.
38+
Expected<std::string> getDefaultDebuginfodCacheDirectory();
39+
40+
/// Finds a default timeout for debuginfod HTTP requests. Checks
41+
/// DEBUGINFOD_TIMEOUT environment variable, default is 90 seconds (90000 ms).
42+
std::chrono::milliseconds getDefaultDebuginfodTimeout();
43+
44+
/// Fetches a specified source file by searching the default local cache
45+
/// directory and server URLs.
46+
Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
47+
StringRef SourceFilePath);
48+
49+
/// Fetches an executable by searching the default local cache directory and
50+
/// server URLs.
51+
Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID);
52+
53+
/// Fetches a debug binary by searching the default local cache directory and
54+
/// server URLs.
55+
Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID);
56+
57+
/// Fetches any debuginfod artifact using the default local cache directory and
58+
/// server URLs.
59+
Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
60+
StringRef UrlPath);
61+
62+
/// Fetches any debuginfod artifact using the specified local cache directory,
63+
/// server URLs, and request timeout (in milliseconds). If the artifact is
64+
/// found, uses the UniqueKey for the local cache file.
65+
Expected<std::string> getCachedOrDownloadArtifact(
66+
StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
67+
ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout);
68+
69+
} // end namespace llvm
70+
71+
#endif

‎llvm/include/llvm/Support/Caching.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,10 @@ using AddBufferFn =
6363
/// the cache directory if it does not already exist. The cache name appears in
6464
/// error messages for errors during caching. The temporary file prefix is used
6565
/// in the temporary file naming scheme used when writing files atomically.
66-
Expected<FileCache> localCache(Twine CacheNameRef, Twine TempFilePrefixRef,
67-
Twine CacheDirectoryPathRef,
68-
AddBufferFn AddBuffer);
66+
Expected<FileCache> localCache(
67+
Twine CacheNameRef, Twine TempFilePrefixRef, Twine CacheDirectoryPathRef,
68+
AddBufferFn AddBuffer = [](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
69+
});
6970
} // namespace llvm
7071

7172
#endif

‎llvm/lib/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ add_subdirectory(Object)
2525
add_subdirectory(ObjectYAML)
2626
add_subdirectory(Option)
2727
add_subdirectory(Remarks)
28+
add_subdirectory(Debuginfod)
2829
add_subdirectory(DebugInfo)
2930
add_subdirectory(DWP)
3031
add_subdirectory(ExecutionEngine)

‎llvm/lib/Debuginfod/CMakeLists.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
add_llvm_component_library(LLVMDebuginfod
2+
Debuginfod.cpp
3+
4+
ADDITIONAL_HEADER_DIRS
5+
${LLVM_MAIN_INCLUDE_DIR}/llvm/Debuginfod
6+
7+
LINK_COMPONENTS
8+
Support
9+
)

‎llvm/lib/Debuginfod/Debuginfod.cpp

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
//===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
///
9+
/// \file
10+
///
11+
/// This file defines the fetchInfo function, which retrieves
12+
/// any of the three supported artifact types: (executable, debuginfo, source
13+
/// file) associated with a build-id from debuginfod servers. If a source file
14+
/// is to be fetched, its absolute path must be specified in the Description
15+
/// argument to fetchInfo.
16+
///
17+
//===----------------------------------------------------------------------===//
18+
19+
#include "llvm/Debuginfod/Debuginfod.h"
20+
#include "llvm/ADT/StringRef.h"
21+
#include "llvm/Support/CachePruning.h"
22+
#include "llvm/Support/Caching.h"
23+
#include "llvm/Support/Error.h"
24+
#include "llvm/Support/FileUtilities.h"
25+
#include "llvm/Support/HTTPClient.h"
26+
#include "llvm/Support/xxhash.h"
27+
28+
namespace llvm {
29+
static std::string uniqueKey(llvm::StringRef S) { return utostr(xxHash64(S)); }
30+
31+
// Returns a binary BuildID as a normalized hex string.
32+
// Uses lowercase for compatibility with common debuginfod servers.
33+
static std::string buildIDToString(BuildIDRef ID) {
34+
return llvm::toHex(ID, /*LowerCase=*/true);
35+
}
36+
37+
Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls() {
38+
const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS");
39+
if (DebuginfodUrlsEnv == NULL)
40+
return SmallVector<StringRef>();
41+
42+
SmallVector<StringRef> DebuginfodUrls;
43+
StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " ");
44+
return DebuginfodUrls;
45+
}
46+
47+
Expected<std::string> getDefaultDebuginfodCacheDirectory() {
48+
if (const char *CacheDirectoryEnv = std::getenv("DEBUGINFOD_CACHE_PATH"))
49+
return CacheDirectoryEnv;
50+
51+
SmallString<64> CacheDirectory;
52+
if (!sys::path::cache_directory(CacheDirectory))
53+
return createStringError(
54+
errc::io_error, "Unable to determine appropriate cache directory.");
55+
return std::string(CacheDirectory);
56+
}
57+
58+
std::chrono::milliseconds getDefaultDebuginfodTimeout() {
59+
long Timeout;
60+
const char *DebuginfodTimeoutEnv = std::getenv("DEBUGINFOD_TIMEOUT");
61+
if (DebuginfodTimeoutEnv &&
62+
to_integer(StringRef(DebuginfodTimeoutEnv).trim(), Timeout, 10))
63+
return std::chrono::milliseconds(Timeout * 1000);
64+
65+
return std::chrono::milliseconds(90 * 1000);
66+
}
67+
68+
/// The following functions fetch a debuginfod artifact to a file in a local
69+
/// cache and return the cached file path. They first search the local cache,
70+
/// followed by the debuginfod servers.
71+
72+
Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
73+
StringRef SourceFilePath) {
74+
SmallString<64> UrlPath;
75+
sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
76+
buildIDToString(ID), "source",
77+
sys::path::convert_to_slash(SourceFilePath));
78+
return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
79+
}
80+
81+
Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID) {
82+
SmallString<64> UrlPath;
83+
sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
84+
buildIDToString(ID), "executable");
85+
return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
86+
}
87+
88+
Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID) {
89+
SmallString<64> UrlPath;
90+
sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
91+
buildIDToString(ID), "debuginfo");
92+
return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
93+
}
94+
95+
// General fetching function.
96+
Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
97+
StringRef UrlPath) {
98+
SmallString<10> CacheDir;
99+
100+
Expected<std::string> CacheDirOrErr = getDefaultDebuginfodCacheDirectory();
101+
if (!CacheDirOrErr)
102+
return CacheDirOrErr.takeError();
103+
CacheDir = *CacheDirOrErr;
104+
105+
Expected<SmallVector<StringRef>> DebuginfodUrlsOrErr =
106+
getDefaultDebuginfodUrls();
107+
if (!DebuginfodUrlsOrErr)
108+
return DebuginfodUrlsOrErr.takeError();
109+
SmallVector<StringRef> &DebuginfodUrls = *DebuginfodUrlsOrErr;
110+
return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir,
111+
DebuginfodUrls,
112+
getDefaultDebuginfodTimeout());
113+
}
114+
115+
Expected<std::string> getCachedOrDownloadArtifact(
116+
StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
117+
ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout) {
118+
SmallString<64> AbsCachedArtifactPath;
119+
sys::path::append(AbsCachedArtifactPath, CacheDirectoryPath,
120+
"llvmcache-" + UniqueKey);
121+
122+
Expected<FileCache> CacheOrErr =
123+
localCache("Debuginfod-client", ".debuginfod-client", CacheDirectoryPath);
124+
if (!CacheOrErr)
125+
return CacheOrErr.takeError();
126+
127+
FileCache Cache = *CacheOrErr;
128+
// We choose an arbitrary Task parameter as we do not make use of it.
129+
unsigned Task = 0;
130+
Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, UniqueKey);
131+
if (!CacheAddStreamOrErr)
132+
return CacheAddStreamOrErr.takeError();
133+
AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
134+
if (!CacheAddStream)
135+
return std::string(AbsCachedArtifactPath);
136+
// The artifact was not found in the local cache, query the debuginfod
137+
// servers.
138+
if (!HTTPClient::isAvailable())
139+
return createStringError(errc::io_error,
140+
"No working HTTP client is available.");
141+
142+
HTTPClient Client;
143+
Client.setTimeout(Timeout);
144+
for (StringRef ServerUrl : DebuginfodUrls) {
145+
SmallString<64> ArtifactUrl;
146+
sys::path::append(ArtifactUrl, sys::path::Style::posix, ServerUrl, UrlPath);
147+
148+
Expected<HTTPResponseBuffer> ResponseOrErr = Client.get(ArtifactUrl);
149+
if (!ResponseOrErr)
150+
return ResponseOrErr.takeError();
151+
152+
HTTPResponseBuffer &Response = *ResponseOrErr;
153+
if (Response.Code != 200)
154+
continue;
155+
156+
// We have retrieved the artifact from this server, and now add it to the
157+
// file cache.
158+
Expected<std::unique_ptr<CachedFileStream>> FileStreamOrErr =
159+
CacheAddStream(Task);
160+
if (FileStreamOrErr)
161+
return FileStreamOrErr.takeError();
162+
std::unique_ptr<CachedFileStream> &FileStream = *FileStreamOrErr;
163+
if (!Response.Body)
164+
return createStringError(
165+
errc::io_error, "Unallocated MemoryBuffer in HTTPResponseBuffer.");
166+
167+
*FileStream->OS << StringRef(Response.Body->getBufferStart(),
168+
Response.Body->getBufferSize());
169+
170+
// Return the path to the artifact on disk.
171+
return std::string(AbsCachedArtifactPath);
172+
}
173+
174+
return createStringError(errc::argument_out_of_domain, "build id not found");
175+
}
176+
} // namespace llvm

‎llvm/unittests/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ add_subdirectory(Bitcode)
2222
add_subdirectory(Bitstream)
2323
add_subdirectory(CodeGen)
2424
add_subdirectory(DebugInfo)
25+
add_subdirectory(Debuginfod)
2526
add_subdirectory(Demangle)
2627
add_subdirectory(ExecutionEngine)
2728
add_subdirectory(FileCheck)

‎llvm/unittests/Debuginfod/CMakeLists.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
set(LLVM_LINK_COMPONENTS
2+
Debuginfod
3+
)
4+
5+
add_llvm_unittest(DebuginfodTests
6+
DebuginfodTests.cpp
7+
)
8+
9+
target_link_libraries(DebuginfodTests PRIVATE LLVMTestingSupport)

‎llvm/unittests/Debuginfod/DebuginfodTests.cpp

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
//===-- llvm/unittest/Support/DebuginfodTests.cpp - unit tests --*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "llvm/Debuginfod/Debuginfod.h"
10+
#include "llvm/Support/FileSystem.h"
11+
#include "llvm/Support/HTTPClient.h"
12+
#include "llvm/Support/Path.h"
13+
#include "llvm/Testing/Support/Error.h"
14+
#include "gtest/gtest.h"
15+
16+
using namespace llvm;
17+
18+
// Check that the Debuginfod client can find locally cached artifacts.
19+
TEST(DebuginfodClient, CacheHit) {
20+
int FD;
21+
SmallString<64> CachedFilePath;
22+
sys::fs::createTemporaryFile("llvmcache-key", "temp", FD, CachedFilePath);
23+
StringRef CacheDir = sys::path::parent_path(CachedFilePath);
24+
StringRef UniqueKey = sys::path::filename(CachedFilePath);
25+
EXPECT_TRUE(UniqueKey.consume_front("llvmcache-"));
26+
raw_fd_ostream OF(FD, true, /*unbuffered=*/true);
27+
OF << "contents\n";
28+
OF << CacheDir << "\n";
29+
OF.close();
30+
Expected<std::string> PathOrErr = getCachedOrDownloadArtifact(
31+
UniqueKey, /*UrlPath=*/"/null", CacheDir,
32+
/*DebuginfodUrls=*/{}, /*Timeout=*/std::chrono::milliseconds(1));
33+
EXPECT_THAT_EXPECTED(PathOrErr, HasValue(CachedFilePath));
34+
}
35+
36+
// Check that the Debuginfod client returns an Error when it fails to find an
37+
// artifact.
38+
TEST(DebuginfodClient, CacheMiss) {
39+
// Ensure there are no urls to guarantee a cache miss.
40+
setenv("DEBUGINFOD_URLS", "", /*replace=*/1);
41+
HTTPClient::initialize();
42+
Expected<std::string> PathOrErr = getCachedOrDownloadArtifact(
43+
/*UniqueKey=*/"nonexistent-key", /*UrlPath=*/"/null");
44+
EXPECT_THAT_EXPECTED(PathOrErr, Failed<StringError>());
45+
}

0 commit comments

Comments
 (0)