[Support] Fix Process::PreventCoreFiles() when coredumps are piped

llvm/lib/Support/Unix/Process.inc

@@ -143,7 +143,26 @@ void Process::GetTimeUsage(TimePoint<> &elapsed,
 void Process::PreventCoreFiles() {
 #if HAVE_SETRLIMIT
   struct rlimit rlim;
-  rlim.rlim_cur = rlim.rlim_max = 0;
+  getrlimit(RLIMIT_CORE, &rlim);
+#ifdef __linux__
+  // On Linux, if the kernel.core_pattern sysctl starts with a '|' (i.e. it
+  // is being piped to a coredump handler such as systemd-coredumpd), the
+  // kernel ignores RLIMIT_CORE (since we aren't creating a file in the file
+  // system) except for the magic value of 1, which disables coredumps when
+  // piping. 1 byte is too small for any kind of valid core dump, so it
+  // also disables coredumps if kernel.core_pattern creates files directly.
+  // While most piped coredump handlers do respect the crashing processes'
+  // RLIMIT_CORE, this is notable not the case for Debian's systemd-coredump
+  // due to a local patch that changes sysctl.d/50-coredump.conf to ignore
+  // the specified limit and instead use RLIM_INFINITY.
+  //
+  // The alternative to using RLIMIT_CORE=1 would be to use prctl() with the
+  // PR_SET_DUMPABLE flag, however that also prevents ptrace(), so makes it
+  // impossible to attach a debugger.
+  rlim.rlim_cur = std::min<rlim_t>(1, rlim.rlim_max);
+#else
+  rlim.rlim_cur = 0;
+#endif
   setrlimit(RLIMIT_CORE, &rlim);
 #endif