Reland: [llvm-cov] Look up object files using debuginfod

Reviewed By: gulfem

Differential Revision: https://reviews.llvm.org/D136702

Author: mysterymath

compiler-rt/test/profile/CMakeLists.txt

@@ -16,6 +16,8 @@ if(APPLE)
   darwin_filter_host_archs(PROFILE_SUPPORTED_ARCH PROFILE_TEST_ARCH)
 endif()
 
+pythonize_bool(LLVM_ENABLE_CURL)
+
 foreach(arch ${PROFILE_TEST_ARCH})
   set(PROFILE_TEST_TARGET_ARCH ${arch})
   get_test_cc_for_arch(${arch} PROFILE_TEST_TARGET_CC PROFILE_TEST_TARGET_CFLAGS)

compiler-rt/test/profile/Linux/binary-id-debuginfod.c

@@ -0,0 +1,33 @@
+// REQUIRES: linux, curl
+// RUN: split-file %s %t
+// RUN: %clang_profgen -Wl,--build-id=0x12345678 -fcoverage-mapping -O2 -shared %t/foo.c -o %t/libfoo.so
+// RUN: %clang_profgen -Wl,--build-id=0xabcd1234 -fcoverage-mapping -O2 %t/main.c -L%t -lfoo -o %t.main
+// RUN: rm -rf %t.profdir
+// RUN: env LLVM_PROFILE_FILE=%t.profdir/default_%m.profraw LD_LIBRARY_PATH=%t %run %t.main
+// RUN: mkdir -p %t/buildid/12345678 %t/buildid/abcd1234
+// RUN: cp %t/libfoo.so %t/buildid/12345678/debuginfo
+// RUN: cp %t.main %t/buildid/abcd1234/debuginfo
+// RUN: llvm-profdata merge -o %t.profdata %t.profdir/default_*.profraw
+// RUN: env DEBUGINFOD_URLS=file://%t llvm-cov show -instr-profile %t.profdata | FileCheck %s
+// RUN: echo "bad" > %t/libfoo.so %t/buildid/12345678/debuginfo
+// RUN: echo "bad" > %t/buildid/abcd1234/debuginfo
+// RUN: env DEBUGINFOD_URLS=file://%t llvm-cov show -instr-profile %t.profdata -debuginfod=false %t.main | FileCheck %s --check-prefix=NODEBUGINFOD
+
+// CHECK: 1| 1|void foo(void) {}
+// CHECK: 2| 1|void bar(void) {}
+// CHECK: 3| 1|int main() {
+
+// NODEBUGINFOD-NOT: foo(void) {}
+// NODEBUGINFOD: main
+
+//--- foo.c
+void foo(void) {}
+
+//--- main.c
+void foo(void);
+void bar(void) {}
+int main() {
+  foo();
+  bar();
+  return 0;
+}

compiler-rt/test/profile/Linux/binary-id-lookup.c

@@ -0,0 +1,32 @@
+// REQUIRES: linux
+// RUN: split-file %s %t
+// RUN: %clang_profgen -Wl,--build-id=0x12345678 -fcoverage-mapping -O2 -shared %t/foo.c -o %t/libfoo.so
+// RUN: %clang_profgen -Wl,--build-id=0xabcd1234 -fcoverage-mapping -O2 %t/main.c -L%t -lfoo -o %t.main
+// RUN: rm -rf %t.profdir
+// RUN: env LLVM_PROFILE_FILE=%t.profdir/default_%m.profraw LD_LIBRARY_PATH=%t %run %t.main
+// RUN: mkdir -p %t/.build-id/12 %t/.build-id/ab
+// RUN: cp %t/libfoo.so %t/.build-id/12/345678.debug
+// RUN: cp %t.main %t/.build-id/ab/cd1234.debug
+// RUN: llvm-profdata merge -o %t.profdata %t.profdir/default_*.profraw
+// RUN: llvm-cov show -instr-profile %t.profdata -debug-file-directory %t | FileCheck %s
+// RUN: echo "bad" > %t/.build-id/ab/cd1234.debug
+// RUN: llvm-cov show -instr-profile %t.profdata -debug-file-directory %t %t.main | FileCheck %s
+// RUN: not llvm-cov show -instr-profile %t.profdata -debug-file-directory %t/empty 2>&1 | FileCheck %s --check-prefix=NODATA
+
+// CHECK: 1| 1|void foo(void) {}
+// CHECK: 2| 1|void bar(void) {}
+// CHECK: 3| 1|int main() {
+
+// NODATA: error: Failed to load coverage: '': No coverage data found
+
+//--- foo.c
+void foo(void) {}
+
+//--- main.c
+void foo(void);
+void bar(void) {}
+int main() {
+  foo();
+  bar();
+  return 0;
+}

compiler-rt/test/profile/Linux/lit.local.cfg.py

@@ -38,3 +38,6 @@ def is_gold_linker_available():
 
 if root.host_os not in ['Linux'] or not is_gold_linker_available():
   config.unsupported = True
+
+if config.have_curl:
+    config.available_features.add('curl')

compiler-rt/test/profile/lit.site.cfg.py.in

@@ -4,6 +4,7 @@
 config.profile_lit_binary_dir = "@PROFILE_LIT_BINARY_DIR@"
 config.target_cflags = "@PROFILE_TEST_TARGET_CFLAGS@"
 config.target_arch = "@PROFILE_TEST_TARGET_ARCH@"
+config.have_curl = @LLVM_ENABLE_CURL_PYBOOL@
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")

llvm/cmake/modules/LLVMConfig.cmake.in

@@ -83,6 +83,11 @@ if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
 endif()
 
+set(LLVM_ENABLE_CURL @LLVM_ENABLE_CURL@)
+if(LLVM_ENABLE_CURL)
+  find_package(CURL)
+endif()
+
 set(LLVM_WITH_Z3 @LLVM_WITH_Z3@)
 
 set(LLVM_ENABLE_DIA_SDK @LLVM_ENABLE_DIA_SDK@)

llvm/docs/CommandGuide/llvm-cov.rst

@@ -349,6 +349,17 @@ OPTIONS
  coverage >= high, red when coverage < low, and yellow otherwise. Both high and
  low should be between 0-100 and high > low.
 
+.. option:: -debuginfod
+
+Use debuginfod to look up coverage mapping for binary IDs present in the profile
+but not in any object given on the command line. Defaults to true if debuginfod
+is compiled in and configured via the DEBUGINFOD_URLS environment variable.
+
+.. option:: -debug-file-directory=<dir>
+
+Provides local directories to search for objects corresponding to binary IDs in
+the profile (as with debuginfod). Defaults to system build ID directories.
+
 .. program:: llvm-cov report
 
 .. _llvm-cov-report:
@@ -418,6 +429,18 @@ OPTIONS
  when binaries have been compiled with one of `-fcoverage-prefix-map`
  `-fcoverage-compilation-dir`, or `-ffile-compilation-dir`.
 
+.. option:: -debuginfod
+
+Attempt to look up coverage mapping from objects using debuginfod. This is
+attempted by default for binary IDs present in the profile but not provided on
+the command line, so long as debuginfod is compiled in and configured via
+DEBUGINFOD_URLS.
+
+.. option:: -debug-file-directory=<dir>
+
+Provides a directory to search for objects corresponding to binary IDs in the
+profile.
+
 .. program:: llvm-cov export
 
 .. _llvm-cov-export:
@@ -492,3 +515,15 @@ OPTIONS
  Directory used as a base for relative coverage mapping paths. Only applicable
  when binaries have been compiled with one of `-fcoverage-prefix-map`
  `-fcoverage-compilation-dir`, or `-ffile-compilation-dir`.
+
+.. option:: -debuginfod
+
+Attempt to look up coverage mapping from objects using debuginfod. This is
+attempted by default for binary IDs present in the profile but not provided on
+the command line, so long as debuginfod is compiled in and configured via
+DEBUGINFOD_URLS.
+
+.. option:: -debug-file-directory=<dir>
+
+Provides a directory to search for objects corresponding to binary IDs in the
+profile.

llvm/include/llvm/Debuginfod/Debuginfod.h

@@ -38,9 +38,13 @@
 
 namespace llvm {
 
+/// Returns false if a debuginfod lookup can be determined to have no chance of
+/// succeeding.
+bool canUseDebuginfod();
+
 /// Finds default array of Debuginfod server URLs by checking DEBUGINFOD_URLS
 /// environment variable.
-Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls();
+SmallVector<StringRef> getDefaultDebuginfodUrls();
 
 /// Finds a default local file caching directory for the debuginfod client,
 /// first checking DEBUGINFOD_CACHE_PATH.

llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h

@@ -21,6 +21,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/Object/BuildID.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Compiler.h"
@@ -42,6 +43,10 @@ namespace llvm {
 
 class IndexedInstrProfReader;
 
+namespace object {
+class BuildIDFetcher;
+} // namespace object
+
 namespace coverage {
 
 class CoverageMappingReader;
@@ -579,6 +584,13 @@ class CoverageMapping {
       ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders,
       IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage);
 
+  // Load coverage records from file.
+  static Error
+  loadFromFile(StringRef Filename, StringRef Arch, StringRef CompilationDir,
+               IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage,
+               bool &DataFound,
+               SmallVectorImpl<object::BuildID> *FoundBinaryIDs = nullptr);
+
   /// Add a function record corresponding to \p Record.
   Error loadFunctionRecord(const CoverageMappingRecord &Record,
                            IndexedInstrProfReader &ProfileReader);
@@ -604,8 +616,8 @@ class CoverageMapping {
   /// Ignores non-instrumented object files unless all are not instrumented.
   static Expected<std::unique_ptr<CoverageMapping>>
   load(ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename,
-       ArrayRef<StringRef> Arches = std::nullopt,
-       StringRef CompilationDir = "");
+       ArrayRef<StringRef> Arches = std::nullopt, StringRef CompilationDir = "",
+       const object::BuildIDFetcher *BIDFetcher = nullptr);
 
   /// The number of functions that couldn't have their profiles mapped.
   ///

llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h

@@ -205,7 +205,8 @@ class BinaryCoverageReader : public CoverageMappingReader {
   static Expected<std::vector<std::unique_ptr<BinaryCoverageReader>>>
   create(MemoryBufferRef ObjectBuffer, StringRef Arch,
          SmallVectorImpl<std::unique_ptr<MemoryBuffer>> &ObjectFileBuffers,
-         StringRef CompilationDir = "");
+         StringRef CompilationDir = "",
+         SmallVectorImpl<object::BuildIDRef> *BinaryIDs = nullptr);
 
   static Expected<std::unique_ptr<BinaryCoverageReader>>
   createCoverageReaderFromBuffer(StringRef Coverage,

llvm/lib/Debuginfod/Debuginfod.cpp

@@ -55,7 +55,11 @@ static std::string buildIDToString(BuildIDRef ID) {
   return llvm::toHex(ID, /*LowerCase=*/true);
 }
 
-Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls() {
+bool canUseDebuginfod() {
+  return HTTPClient::isAvailable() && !getDefaultDebuginfodUrls().empty();
+}
+
+SmallVector<StringRef> getDefaultDebuginfodUrls() {
   const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS");
   if (DebuginfodUrlsEnv == nullptr)
     return SmallVector<StringRef>();
@@ -126,13 +130,8 @@ Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
     return CacheDirOrErr.takeError();
   CacheDir = *CacheDirOrErr;
 
-  Expected<SmallVector<StringRef>> DebuginfodUrlsOrErr =
-      getDefaultDebuginfodUrls();
-  if (!DebuginfodUrlsOrErr)
-    return DebuginfodUrlsOrErr.takeError();
-  SmallVector<StringRef> &DebuginfodUrls = *DebuginfodUrlsOrErr;
   return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir,
-                                     DebuginfodUrls,
+                                     getDefaultDebuginfodUrls(),
                                      getDefaultDebuginfodTimeout());
 }
 
@@ -159,7 +158,8 @@ class StreamedHTTPResponseHandler : public HTTPResponseHandler {
 
 Error StreamedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) {
   if (!FileStream) {
-    if (Client.responseCode() != 200)
+    unsigned Code = Client.responseCode();
+    if (Code && Code != 200)
       return Error::success();
     Expected<std::unique_ptr<CachedFileStream>> FileStreamOrError =
         CreateStream();
@@ -259,7 +259,8 @@ Expected<std::string> getCachedOrDownloadArtifact(
     if (Err)
       return std::move(Err);
 
-    if (Client.responseCode() != 200)
+    unsigned Code = Client.responseCode();
+    if (Code && Code != 200)
       continue;
 
     // Return the path to the artifact on disk.