[profile] Support counter relocation at runtime

This is an alternative to the continous mode that was implemented in
D68351. This mode relies on padding and the ability to mmap a file over
the existing mapping which is generally only available on POSIX systems
and isn't suitable for other platforms.

This change instead introduces the ability to relocate counters at
runtime using a level of indirection. On every counter access, we add a
bias to the counter address. This bias is stored in a symbol that's
provided by the profile runtime and is initially set to zero, meaning no
relocation. The runtime can mmap the profile into memory at abitrary
location, and set bias to the offset between the original and the new
counter location, at which point every subsequent counter access will be
to the new location, which allows updating profile directly akin to the
continous mode.

The advantage of this implementation is that doesn't require any special
OS support. The disadvantage is the extra overhead due to additional
instructions required for each counter access (overhead both in terms of
binary size and performance) plus duplication of counters (i.e. one copy
in the binary itself and another copy that's mmapped).

Differential Revision: https://reviews.llvm.org/D69740

Author: petrhosek

clang/docs/SourceBasedCodeCoverage.rst

@@ -92,15 +92,42 @@ directory structure will be created.  Additionally, the following special
   instrumented program crashes, or is killed by a signal, perfect coverage
   information can still be recovered. Continuous mode does not support value
   profiling for PGO, and is only supported on Darwin at the moment. Support for
-  Linux may be mostly complete but requires testing, and support for
-  Fuchsia/Windows may require more extensive changes: please get involved if
-  you are interested in porting this feature.
+  Linux may be mostly complete but requires testing, and support for Windows
+  may require more extensive changes: please get involved if you are interested
+  in porting this feature.
 
 .. code-block:: console
 
     # Step 2: Run the program.
     % LLVM_PROFILE_FILE="foo.profraw" ./foo
 
+Note that continuous mode is also used on Fuchsia where it's the only supported
+mode, but the implementation is different. The Darwin and Linux implementation
+relies on padding and the ability to map a file over the existing memory
+mapping which is generally only available on POSIX systems and isn't suitable
+for other platforms.
+
+On Fuchsia, we rely on the the ability to relocate counters at runtime using a
+level of indirection. On every counter access, we add a bias to the counter
+address. This bias is stored in ``__llvm_profile_counter_bias`` symbol that's
+provided by the profile runtime and is initially set to zero, meaning no
+relocation. The runtime can map the profile into memory at abitrary location,
+and set bias to the offset between the original and the new counter location,
+at which point every subsequent counter access will be to the new location,
+which allows updating profile directly akin to the continous mode.
+
+The advantage of this approach is that doesn't require any special OS support.
+The disadvantage is the extra overhead due to additional instructions required
+for each counter access (overhead both in terms of binary size and performance)
+plus duplication of counters (i.e. one copy in the binary itself and another
+copy that's mapped into memory). This implementation can be also enabled for
+other platforms by passing the ``-runtime-counter-relocation`` option to the
+backend during compilation.
+
+.. code-block:: console
+
+    % clang++ -fprofile-instr-generate -fcoverage-mapping -mllvm -runtime-counter-relocation foo.cc -o foo
+
 Creating coverage reports
 =========================
 

clang/lib/Driver/ToolChains/Darwin.cpp

@@ -1149,6 +1149,7 @@ void Darwin::addProfileRTLibs(const ArgList &Args,
     } else {
       addExportedSymbol(CmdArgs, "___llvm_profile_filename");
       addExportedSymbol(CmdArgs, "___llvm_profile_raw_version");
+      addExportedSymbol(CmdArgs, "___llvm_profile_counter_bias");
     }
     addExportedSymbol(CmdArgs, "_lprofDirMode");
   }

compiler-rt/lib/profile/CMakeLists.txt

@@ -52,6 +52,7 @@ set(PROFILE_SOURCES
   GCDAProfiling.c
   InstrProfiling.c
   InstrProfilingValue.c
+  InstrProfilingBiasVar.c
   InstrProfilingBuffer.c
   InstrProfilingFile.c
   InstrProfilingMerge.c

compiler-rt/lib/profile/InstrProfiling.h

@@ -307,4 +307,11 @@ extern uint64_t INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */
  */
 extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */
 
+/*!
+ * This variable is a weak symbol defined in InstrProfilingBiasVar.c. It
+ * allows compiler instrumentation to provide overriding definition with
+ * value from compiler command line. This variable has hidden visibility.
+ */
+COMPILER_RT_VISIBILITY extern intptr_t __llvm_profile_counter_bias;
+
 #endif /* PROFILE_INSTRPROFILING_H_ */

compiler-rt/lib/profile/InstrProfilingBiasVar.c

@@ -0,0 +1,15 @@
+/*===- InstrProfilingBiasVar.c - profile counter bias variable setup ------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+
+/* The runtime should only provide its own definition of this symbol when the
+ * user has not specified one. Set this up by moving the runtime's copy of this
+ * symbol to an object file within the archive.
+ */
+COMPILER_RT_WEAK intptr_t __llvm_profile_counter_bias = -1;

compiler-rt/lib/profile/InstrProfilingBuffer.c

@@ -10,6 +10,9 @@
 #include "InstrProfilingInternal.h"
 #include "InstrProfilingPort.h"
 
+/* When counters are being relocated at runtime, this parameter is set to 1. */
+COMPILER_RT_VISIBILITY int RuntimeCounterRelocation = 0;
+
 /* When continuous mode is enabled (%c), this parameter is set to 1.
  *
  * This parameter is defined here in InstrProfilingBuffer.o, instead of in
@@ -62,7 +65,8 @@ void __llvm_profile_get_padding_sizes_for_counters(
     uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
     uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
     uint64_t *PaddingBytesAfterNames) {
-  if (!__llvm_profile_is_continuous_mode_enabled()) {
+  if (!__llvm_profile_is_continuous_mode_enabled() ||
+      RuntimeCounterRelocation) {
     *PaddingBytesBeforeCounters = 0;
     *PaddingBytesAfterCounters = 0;
     *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);

compiler-rt/lib/profile/InstrProfilingFile.c

@@ -448,6 +448,98 @@ static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) {
 }
 #endif // !defined(__Fuchsia__) && !defined(_WIN32)
 
+static int writeMMappedFile(FILE *OutputFile, char **Profile) {
+  if (!OutputFile)
+    return -1;
+
+  /* Write the data into a file. */
+  setupIOBuffer();
+  ProfDataWriter fileWriter;
+  initFileWriter(&fileWriter, OutputFile);
+  if (lprofWriteData(&fileWriter, NULL, 0)) {
+    PROF_ERR("Failed to write profile: %s\n", strerror(errno));
+    return -1;
+  }
+  fflush(OutputFile);
+
+  /* Get the file size. */
+  uint64_t FileSize = ftell(OutputFile);
+
+  /* Map the profile. */
+  *Profile = (char *)mmap(
+      NULL, FileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(OutputFile), 0);
+  if (*Profile == MAP_FAILED) {
+    PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
+    return -1;
+  }
+
+  return 0;
+}
+
+static void relocateCounters(void) {
+  if (!__llvm_profile_is_continuous_mode_enabled() || !RuntimeCounterRelocation)
+    return;
+
+  /* Get the sizes of various profile data sections. Taken from
+   * __llvm_profile_get_size_for_buffer(). */
+  const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+  const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+  uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+  const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
+      (DataSize * sizeof(__llvm_profile_data));
+
+  int Length = getCurFilenameLength();
+  char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+  const char *Filename = getCurFilename(FilenameBuf, 0);
+  if (!Filename)
+    return;
+
+  FILE *File = NULL;
+  char *Profile = NULL;
+
+  if (!doMerging()) {
+    File = fopen(Filename, "w+b");
+    if (!File)
+      return;
+
+    if (writeMMappedFile(File, &Profile) == -1) {
+      fclose(File);
+      return;
+    }
+  } else {
+    File = lprofOpenFileEx(Filename);
+    if (!File)
+      return;
+
+    uint64_t ProfileFileSize = 0;
+    if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
+      lprofUnlockFileHandle(File);
+      fclose(File);
+      return;
+    }
+
+    if (!ProfileFileSize) {
+      if (writeMMappedFile(File, &Profile) == -1) {
+        fclose(File);
+        return;
+      }
+    } else {
+      /* The merged profile has a non-zero length. Check that it is compatible
+       * with the data in this process. */
+      if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) {
+        fclose(File);
+        return;
+      }
+    }
+
+    lprofUnlockFileHandle(File);
+  }
+
+  /* Update the profile fields based on the current mapping. */
+  __llvm_profile_counter_bias = (intptr_t)Profile -
+      (uintptr_t)__llvm_profile_begin_counters() + CountersOffset;
+}
+
 static void initializeProfileForContinuousMode(void) {
   if (!__llvm_profile_is_continuous_mode_enabled())
     return;
@@ -715,7 +807,12 @@ static void parseAndSetFilename(const char *FilenamePat,
   }
 
   truncateCurrentFile();
-  initializeProfileForContinuousMode();
+  if (__llvm_profile_is_continuous_mode_enabled()) {
+    if (RuntimeCounterRelocation)
+      relocateCounters();
+    else
+      initializeProfileForContinuousMode();
+  }
 }
 
 /* Return buffer length that is required to store the current profile
@@ -865,6 +962,9 @@ void __llvm_profile_initialize_file(void) {
   ProfileNameSpecifier PNS = PNS_unknown;
   int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
 
+  if (__llvm_profile_counter_bias != -1)
+    RuntimeCounterRelocation = 1;
+
   EnvFilenamePat = getFilenamePatFromEnv();
   if (EnvFilenamePat) {
     /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid

compiler-rt/lib/profile/InstrProfilingInternal.h

@@ -184,6 +184,7 @@ uint64_t lprofGetLoadModuleSignature();
 unsigned lprofProfileDumped();
 void lprofSetProfileDumped();
 
+COMPILER_RT_VISIBILITY extern int RuntimeCounterRelocation;
 COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *);
 COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer;
 COMPILER_RT_VISIBILITY extern uint32_t VPBufferSize;