[profile] Add a mode to continuously sync counter updates to a file

Add support for continuously syncing profile counter updates to a file.

The motivation for this is that programs do not always exit cleanly. On
iOS, for example, programs are usually killed via a signal from the OS.
Running atexit() handlers after catching a signal is unreliable, so some
method for progressively writing out profile data is necessary.

The approach taken here is to mmap() the `__llvm_prf_cnts` section onto
a raw profile. To do this, the linker must page-align the counter and
data sections, and the runtime must ensure that counters are mapped to a
page-aligned offset within a raw profile.

Continuous mode is (for the moment) incompatible with the online merging
mode. This limitation is lifted in https://reviews.llvm.org/D69586.

Continuous mode is also (for the moment) incompatible with value
profiling, as I'm not sure whether there is interest in this and the
implementation may be tricky.

As I have not been able to test extensively on non-Darwin platforms,
only Darwin support is included for the moment. However, continuous mode
may "just work" without modification on Linux and some UNIX-likes. AIUI
the default value for the GNU linker's `--section-alignment` flag is set
to the page size on many systems. This appears to be true for LLD as
well, as its `no_nmagic` option is on by default. Continuous mode will
not "just work" on Fuchsia or Windows, as it's not possible to mmap() a
section on these platforms. There is a proposal to add a layer of
indirection to the profile instrumentation to support these platforms.

rdar://54210980

Differential Revision: https://reviews.llvm.org/D68351

Author: vedantk

clang/docs/SourceBasedCodeCoverage.rst

@@ -87,6 +87,16 @@ directory structure will be created.  Additionally, the following special
   be between 1 and 9. The merge pool specifier can only occur once per filename
   pattern.
 
+* "%c" expands out to nothing, but enables a mode in which profile counter
+  updates are continuously synced to a file. This means that if the
+  instrumented program crashes, or is killed by a signal, perfect coverage
+  information can still be recovered. Continuous mode is not yet compatible with
+  the "%Nm" merging mode described above, does not support value profiling for
+  PGO, and is only supported on Darwin. Support for Linux may be mostly
+  complete but requires testing, and support for Fuchsia/Windows may require
+  more extensive changes: please get involved if you are interested in porting
+  this feature.
+
 .. code-block:: console
 
     # Step 2: Run the program.

clang/lib/Driver/ToolChains/Darwin.cpp

@@ -19,6 +19,7 @@
 #include "clang/Driver/SanitizerArgs.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Option/ArgList.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/TargetParser.h"
@@ -1110,18 +1111,33 @@ static void addExportedSymbol(ArgStringList &CmdArgs, const char *Symbol) {
   CmdArgs.push_back(Symbol);
 }
 
+/// Add a sectalign directive for \p Segment and \p Section to the maximum
+/// expected page size for Darwin.
+///
+/// On iPhone 6+ the max supported page size is 16K. On macOS, the max is 4K.
+/// Use a common alignment constant (16K) for now, and reduce the alignment on
+/// macOS if it proves important.
+static void addSectalignToPage(const ArgList &Args, ArgStringList &CmdArgs,
+                               StringRef Segment, StringRef Section) {
+  for (const char *A : {"-sectalign", Args.MakeArgString(Segment),
+                        Args.MakeArgString(Section), "0x4000"})
+    CmdArgs.push_back(A);
+}
+
 void Darwin::addProfileRTLibs(const ArgList &Args,
                               ArgStringList &CmdArgs) const {
   if (!needsProfileRT(Args)) return;
 
   AddLinkRuntimeLib(Args, CmdArgs, "profile",
                     RuntimeLinkOptions(RLO_AlwaysLink | RLO_FirstLink));
 
+  bool ForGCOV = needsGCovInstrumentation(Args);
+
   // If we have a symbol export directive and we're linking in the profile
   // runtime, automatically export symbols necessary to implement some of the
   // runtime's functionality.
   if (hasExportSymbolDirective(Args)) {
-    if (needsGCovInstrumentation(Args)) {
+    if (ForGCOV) {
       addExportedSymbol(CmdArgs, "___gcov_flush");
       addExportedSymbol(CmdArgs, "_flush_fn_list");
       addExportedSymbol(CmdArgs, "_writeout_fn_list");
@@ -1131,6 +1147,24 @@ void Darwin::addProfileRTLibs(const ArgList &Args,
     }
     addExportedSymbol(CmdArgs, "_lprofDirMode");
   }
+
+  // Align __llvm_prf_{cnts,data} sections to the maximum expected page
+  // alignment. This allows profile counters to be mmap()'d to disk. Note that
+  // it's not enough to just page-align __llvm_prf_cnts: the following section
+  // must also be page-aligned so that its data is not clobbered by mmap().
+  //
+  // The section alignment is only needed when continuous profile sync is
+  // enabled, but this is expected to be the default in Xcode. Specifying the
+  // extra alignment also allows the same binary to be used with/without sync
+  // enabled.
+  if (!ForGCOV) {
+    for (auto IPSK : {llvm::IPSK_cnts, llvm::IPSK_data}) {
+      addSectalignToPage(
+          Args, CmdArgs, "__DATA",
+          llvm::getInstrProfSectionName(IPSK, llvm::Triple::MachO,
+                                        /*AddSegmentInfo=*/false));
+    }
+  }
 }
 
 void DarwinClang::AddLinkSanitizerLibArgs(const ArgList &Args,

clang/test/Driver/darwin-ld.c

@@ -345,6 +345,12 @@
 // RUN: FileCheck -check-prefix=LINK_PROFILE_FIRST %s < %t.log
 // LINK_PROFILE_FIRST: {{ld(.exe)?"}} "{{[^"]+}}libclang_rt.profile_{{[a-z]+}}.a"
 
+// RUN: %clang -target x86_64-apple-darwin12 -fprofile-instr-generate -### %t.o 2> %t.log
+// RUN: FileCheck -check-prefix=PROFILE_SECTALIGN %s < %t.log
+// RUN: %clang -target arm64-apple-ios12 -fprofile-instr-generate -### %t.o 2> %t.log
+// RUN: FileCheck -check-prefix=PROFILE_SECTALIGN %s < %t.log
+// PROFILE_SECTALIGN: "-sectalign" "__DATA" "__llvm_prf_cnts" "0x4000" "-sectalign" "__DATA" "__llvm_prf_data" "0x4000"
+
 // RUN: %clang -target x86_64-apple-darwin12 -fprofile-instr-generate -exported_symbols_list /dev/null -### %t.o 2> %t.log
 // RUN: FileCheck -check-prefix=PROFILE_EXPORT %s < %t.log
 // RUN: %clang -target x86_64-apple-darwin12 -fprofile-instr-generate -Wl,-exported_symbols_list,/dev/null -### %t.o 2> %t.log

compiler-rt/lib/profile/InstrProfData.inc

@@ -130,7 +130,9 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
 INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
 INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
 INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
+INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
 INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
+INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
 INSTR_PROF_RAW_HEADER(uint64_t, NamesSize,  NamesSize)
 INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
 INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
@@ -628,7 +630,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
         (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
 
 /* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 4
+#define INSTR_PROF_RAW_VERSION 5
 /* Indexed profile format version (start from 1). */
 #define INSTR_PROF_INDEX_VERSION 5
 /* Coverage mapping format vresion (start from 0). */

compiler-rt/lib/profile/InstrProfiling.h

@@ -38,6 +38,22 @@ typedef struct ValueProfNode {
 #include "InstrProfData.inc"
 } ValueProfNode;
 
+/*!
+ * \brief Return 1 if profile counters are continuously synced to the raw
+ * profile via an mmap(). This is in contrast to the default mode, in which
+ * the raw profile is written out at program exit time.
+ */
+int __llvm_profile_is_continuous_mode_enabled(void);
+
+/*!
+ * \brief Enable continuous mode.
+ *
+ * See \ref __llvm_profile_is_continuous_mode_enabled. The behavior is undefined
+ * if continuous mode is already enabled, or if it cannot be enable due to
+ * conflicting options.
+ */
+void __llvm_profile_enable_continuous_mode(void);
+
 /*!
  * \brief Get number of bytes necessary to pad the argument to eight
  * byte boundary.
@@ -159,6 +175,12 @@ int __llvm_orderfile_dump(void);
  * Note: There may be multiple copies of the profile runtime (one for each
  * instrumented image/DSO). This API only modifies the filename within the
  * copy of the runtime available to the calling image.
+ *
+ * Warning: This is a no-op if continuous mode (\ref
+ * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is
+ * that in continuous mode, profile counters are mmap()'d to the profile at
+ * program initialization time. Support for transferring the mmap'd profile
+ * counts to a new file has not been implemented.
  */
 void __llvm_profile_set_filename(const char *Name);
 
@@ -181,6 +203,12 @@ void __llvm_profile_set_filename(const char *Name);
  * Note: There may be multiple copies of the profile runtime (one for each
  * instrumented image/DSO). This API only modifies the file object within the
  * copy of the runtime available to the calling image.
+ *
+ * Warning: This is a no-op if continuous mode (\ref
+ * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is
+ * that in continuous mode, profile counters are mmap()'d to the profile at
+ * program initialization time. Support for transferring the mmap'd profile
+ * counts to a new file has not been implemented.
  */
 void __llvm_profile_set_file_object(FILE *File, int EnableMerge);
 
@@ -223,6 +251,24 @@ uint64_t __llvm_profile_get_version(void);
 uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
                                       const __llvm_profile_data *End);
 
+/* ! \brief Given the sizes of the data and counter information, return the
+ * number of padding bytes before and after the counters, and after the names,
+ * in the raw profile.
+ *
+ * Note: In this context, "size" means "number of entries", i.e. the first two
+ * arguments must be the result of __llvm_profile_get_data_size() and of
+ * (__llvm_profile_end_counters() - __llvm_profile_begin_counters()) resp.
+ *
+ * Note: When mmap() mode is disabled, no padding bytes before/after counters
+ * are needed. However, in mmap() mode, the counter section in the raw profile
+ * must be page-aligned: this API computes the number of padding bytes
+ * needed to achieve that.
+ */
+void __llvm_profile_get_padding_sizes_for_counters(
+    uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
+    uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
+    uint64_t *PaddingBytesAfterNames);
+
 /*!
  * \brief Set the flag that profile data has been dumped to the file.
  * This is useful for users to disable dumping profile data to the file for

compiler-rt/lib/profile/InstrProfilingBuffer.c

@@ -8,6 +8,27 @@
 
 #include "InstrProfiling.h"
 #include "InstrProfilingInternal.h"
+#include "InstrProfilingPort.h"
+
+/* When continuous mode is enabled (%c), this parameter is set to 1. This is
+ * incompatible with the in-process merging mode. Lifting this restriction
+ * may be complicated, as merging mode requires a lock on the profile, and
+ * mmap() mode would require that lock to be held for the entire process
+ * lifetime.
+ *
+ * This parameter is defined here in InstrProfilingBuffer.o, instead of in
+ * InstrProfilingFile.o, to sequester all libc-dependent code in
+ * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this
+ * layering is violated. */
+static int ContinuouslySyncProfile = 0;
+
+COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {
+  return ContinuouslySyncProfile;
+}
+
+COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {
+  ContinuouslySyncProfile = 1;
+}
 
 COMPILER_RT_VISIBILITY
 uint64_t __llvm_profile_get_size_for_buffer(void) {
@@ -30,18 +51,63 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
          sizeof(__llvm_profile_data);
 }
 
+/// Calculate the number of padding bytes needed to add to \p Offset in order
+/// for (\p Offset + Padding) to be page-aligned.
+static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset,
+                                                unsigned PageSize) {
+  uint64_t OffsetModPage = Offset % PageSize;
+  if (OffsetModPage > 0)
+    return PageSize - OffsetModPage;
+  return 0;
+}
+
+COMPILER_RT_VISIBILITY
+void __llvm_profile_get_padding_sizes_for_counters(
+    uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
+    uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
+    uint64_t *PaddingBytesAfterNames) {
+  if (!__llvm_profile_is_continuous_mode_enabled()) {
+    *PaddingBytesBeforeCounters = 0;
+    *PaddingBytesAfterCounters = 0;
+    *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
+    return;
+  }
+
+  // In continuous mode, the file offsets for headers and for the start of
+  // counter sections need to be page-aligned.
+  unsigned PageSize = getpagesize();
+  uint64_t DataSizeInBytes = DataSize * sizeof(__llvm_profile_data);
+  uint64_t CountersSizeInBytes = CountersSize * sizeof(uint64_t);
+  *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign(
+      sizeof(__llvm_profile_header) + DataSizeInBytes, PageSize);
+  *PaddingBytesAfterCounters =
+      calculateBytesNeededToPageAlign(CountersSizeInBytes, PageSize);
+  *PaddingBytesAfterNames =
+      calculateBytesNeededToPageAlign(NamesSize, PageSize);
+}
+
 COMPILER_RT_VISIBILITY
 uint64_t __llvm_profile_get_size_for_buffer_internal(
     const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
     const uint64_t *CountersBegin, const uint64_t *CountersEnd,
     const char *NamesBegin, const char *NamesEnd) {
   /* Match logic in __llvm_profile_write_buffer(). */
   const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
-  const uint8_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize);
+  uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+  uint64_t CountersSize = CountersEnd - CountersBegin;
+
+  /* Determine how much padding is needed before/after the counters and after
+   * the names. */
+  uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
+      PaddingBytesAfterNames;
+  __llvm_profile_get_padding_sizes_for_counters(
+      DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
+      &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
+
   return sizeof(__llvm_profile_header) +
-         (__llvm_profile_get_data_size(DataBegin, DataEnd) *
-          sizeof(__llvm_profile_data)) +
-         (CountersEnd - CountersBegin) * sizeof(uint64_t) + NamesSize + Padding;
+         (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters +
+         (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters +
+         NamesSize + PaddingBytesAfterNames;
 }
 
 COMPILER_RT_VISIBILITY