Merge pull request #16444 from BenchR267/feature/add-metrics-to-driver-output

[Driver] Added process information to emitted task messages

Author: BenchR267

CMakeLists.txt

@@ -893,6 +893,8 @@ if(HAVE_EL_WGETS)
 endif()
 cmake_pop_check_state()
 
+check_symbol_exists(wait4 "sys/wait.h" HAVE_WAIT4)
+
 if (LLVM_ENABLE_DOXYGEN)
   message(STATUS "Doxygen: enabled")
 endif()

docs/DriverParseableOutput.rst

@@ -55,6 +55,12 @@ real process. By using negative numbers for quasi-PIDs, we ensure that no two
 jobs will have the same PID: no batch mode job can have the same PID as a real
 process. Two jobs with the same PID would violate this uniqueness assumption.
 
+However, if you need to know the real PID of the process, you can use the
+'process.real_pid' field in the began/finished/signalled message. It will contain
+the process identifier the operating system gave the process.
+Be aware that this does not guarantee to be unique as contrasted with "pid"!
+If "pid" is greater than 0 it will always match the value in "process.real_pid".
+
 
 Message Kinds
 =============
@@ -76,13 +82,18 @@ key. An "outputs" object will have two fields, a "kind" describing the type of
 the output, and a "path" containing the path to the output. A "began" message
 will specify the command which was executed under the "command_executable" and
 "command_arguments" keys.
+To get the real process identifier for the process that started, get the value
+of "process.real_pid".
 
 Example::
 
    {
      "kind": "began",
      "name": "compile",
      "pid": 12345,
+     "process": {
+       "real_pid": 12345
+     },
      "inputs": ["/src/foo.swift"],
      "outputs": [
         {
@@ -110,14 +121,24 @@ based messages, it will include the task's PID (or quasi-PID) under the "pid"
 key. It will include the exit status of the task under the "exit-status" key. It
 may include the stdout/stderr of the task under the "output" key; if this key is
 missing, no output was generated by the task.
+It will contain the process identifier of the operating system and usage under
+the "process" key. The usage is optional and could be omitted.
 
 Example::
 
    {
      "kind": "finished",
      "name": "compile",
      "pid": 12345,
-     "exit-status": 0
+     "exit-status": 0,
+     "process": {
+       "real_pid": 12345,
+       "usage": {
+         "utime": 22740,
+         "stime": 91107,
+         "maxrss": 7745536
+       }
+      }
      // "output" key omitted because there was no stdout/stderr.
    }
 
@@ -131,6 +152,8 @@ the "error-message" key. As with the "finished" message, it may include the
 stdout/stderr of the task under the "output" key; if this key is missing, no
 output was generated by the task. It may include the "signal" key, the
 terminating signal number. (This may not be available on all platforms.)
+It will contain the process identifier of the operating system and usage under
+the "process" key. The usage is optional and could be omitted.
 
 Example::
 
@@ -139,7 +162,15 @@ Example::
      "name": "compile",
      "pid": 12345,
      "error-message": "Segmentation fault: 11",
-     "signal": 4
+     "signal": 4,
+     "process": {
+       "real_pid": 12345,
+       "usage": {
+         "utime": 22740,
+         "stime": 91107,
+         "maxrss": 7745536
+       }
+      }
      // "output" key omitted because there was no stdout/stderr.
    }
 

include/swift/Basic/TaskQueue.h

@@ -13,6 +13,7 @@
 #ifndef SWIFT_BASIC_TASKQUEUE_H
 #define SWIFT_BASIC_TASKQUEUE_H
 
+#include "swift/Basic/JSONSerialization.h"
 #include "swift/Basic/LLVM.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Config/config.h"
@@ -22,6 +23,10 @@
 #include <memory>
 #include <queue>
 
+#if defined(HAVE_GETRUSAGE) && !defined(__HAIKU__)
+struct rusage;
+#endif
+
 namespace swift {
 class UnifiedStatsReporter;
 namespace sys {
@@ -39,6 +44,62 @@ enum class TaskFinishedResponse {
   StopExecution,
 };
 
+/// TaskProcessInformation is bound to a task and contains information about the
+/// process that ran this task. This is especially useful to find out which
+/// tasks ran in the same process (in multifile-mode or when WMO is activated
+/// e.g.). If available, it also contains information about the usage of
+/// resources like CPU time or memory the process used in the system. How ever,
+/// this could differ from platform to platform and is therefore optional.
+
+/// One process could handle multiple tasks in some modes of the Swift compiler
+/// (multifile, WMO). To not break existing tools, the driver does use unique
+/// identifiers for the tasks that are not the process identifier. To still be
+/// able to reason about tasks that ran in the same process the
+/// TaskProcessInformation struct contains information about the actual process
+/// of the operating system. The OSPid is the actual process identifier and is
+/// therefore not guaranteed to be unique over all tasks. The ProcessUsage
+/// contains optional usage information about the operating system process. It
+/// could be used by tools that take those information as input for analyzing
+/// the Swift compiler on a process-level. It will be `None` if the execution
+/// has been skipped or one of the following symbols are not available on the
+/// system: `rusage`, `wait4`.
+struct TaskProcessInformation {
+
+  struct ResourceUsage {
+    // user time in µs
+    uint64_t Utime;
+    // system time in µs
+    uint64_t Stime;
+    // maximum resident set size in Bytes
+    uint64_t Maxrss;
+
+    ResourceUsage(uint64_t Utime, uint64_t Stime, uint64_t Maxrss)
+        : Utime(Utime), Stime(Stime), Maxrss(Maxrss) {}
+
+    virtual ~ResourceUsage() = default;
+    virtual void provideMapping(json::Output &out);
+  };
+
+private:
+  // the process identifier of the operating system
+  ProcessId OSPid;
+  // usage information about the process, if available
+  Optional<ResourceUsage> ProcessUsage;
+
+public:
+  TaskProcessInformation(ProcessId Pid, uint64_t utime, uint64_t stime,
+                         uint64_t maxrss)
+      : OSPid(Pid), ProcessUsage(ResourceUsage(utime, stime, maxrss)) {}
+
+  TaskProcessInformation(ProcessId Pid) : OSPid(Pid), ProcessUsage(None) {}
+
+#if defined(HAVE_GETRUSAGE) && !defined(__HAIKU__)
+  TaskProcessInformation(ProcessId Pid, struct rusage Usage);
+#endif // defined(HAVE_GETRUSAGE) && !defined(__HAIKU__)
+  virtual ~TaskProcessInformation() = default;
+  virtual void provideMapping(json::Output &out);
+};
+
 /// \brief A class encapsulating the execution of multiple tasks in parallel.
 class TaskQueue {
   /// Tasks which have not begun execution.
@@ -81,13 +142,15 @@ class TaskQueue {
   /// \param Errors the errors from the task which finished execution, if
   /// available and SeparateErrors was true.  (This may not be available on all
   /// platforms.)
+  /// \param ProcInfo contains information like the operating process identifier
+  /// and resource usage if available
   /// \param Context the context which was passed when the task was added
   ///
   /// \returns true if further execution of tasks should stop,
   /// false if execution should continue
   using TaskFinishedCallback = std::function<TaskFinishedResponse(
       ProcessId Pid, int ReturnCode, StringRef Output, StringRef Errors,
-      void *Context)>;
+      TaskProcessInformation ProcInfo, void *Context)>;
 
   /// \brief A callback which will be executed if a task exited abnormally due
   /// to a signal.
@@ -100,16 +163,18 @@ class TaskQueue {
   /// \param Errors the errors from the task which exited abnormally, if
   /// available and SeparateErrors was true.  (This may not be available on all
   /// platforms.)
+  /// \param ProcInfo contains information like the operating process identifier
+  /// and resource usage if available
   /// \param Context the context which was passed when the task was added
-  /// \param Signal the terminating signal number, if available.
-  /// This may not be available on all platforms. If it is ever provided,
-  /// it should not be removed in future versions of the compiler.
+  /// \param Signal the terminating signal number, if available. This may not be
+  /// available on all platforms. If it is ever provided, it should not be
+  /// removed in future versions of the compiler.
   ///
   /// \returns a TaskFinishedResponse indicating whether or not execution
   /// should proceed
   using TaskSignalledCallback = std::function<TaskFinishedResponse(
       ProcessId Pid, StringRef ErrorMsg, StringRef Output, StringRef Errors,
-      void *Context, Optional<int> Signal)>;
+      void *Context, Optional<int> Signal, TaskProcessInformation ProcInfo)>;
 #pragma clang diagnostic pop
 
   /// \brief Indicates whether TaskQueue supports buffering output on the
@@ -202,6 +267,22 @@ class DummyTaskQueue : public TaskQueue {
 };
 
 } // end namespace sys
+
+namespace json {
+template <> struct ObjectTraits<sys::TaskProcessInformation> {
+  static void mapping(Output &out, sys::TaskProcessInformation &value) {
+    value.provideMapping(out);
+  }
+};
+
+template <> struct ObjectTraits<sys::TaskProcessInformation::ResourceUsage> {
+  static void mapping(Output &out,
+                      sys::TaskProcessInformation::ResourceUsage &value) {
+    value.provideMapping(out);
+  }
+};
+} // end namespace json
+
 } // end namespace swift
 
 #endif // SWIFT_BASIC_TASKQUEUE_H

include/swift/Config.h.in

@@ -8,6 +8,8 @@
 
 #cmakedefine HAVE_UNICODE_LIBEDIT 1
 
+#cmakedefine HAVE_WAIT4 1
+
 #cmakedefine01 SWIFT_DARWIN_ENABLE_STABLE_ABI_BIT
 
 #endif // SWIFT_CONFIG_H

include/swift/Driver/ParseableOutput.h

@@ -29,15 +29,19 @@ class Job;
 namespace parseable_output {
 
 /// \brief Emits a "began" message to the given stream.
-void emitBeganMessage(raw_ostream &os, const Job &Cmd, int64_t Pid);
+void emitBeganMessage(raw_ostream &os, const Job &Cmd, int64_t Pid,
+                      sys::TaskProcessInformation ProcInfo);
 
 /// \brief Emits a "finished" message to the given stream.
 void emitFinishedMessage(raw_ostream &os, const Job &Cmd, int64_t Pid,
-                         int ExitStatus, StringRef Output);
+                         int ExitStatus, StringRef Output,
+                         sys::TaskProcessInformation ProcInfo);
 
 /// \brief Emits a "signalled" message to the given stream.
 void emitSignalledMessage(raw_ostream &os, const Job &Cmd, int64_t Pid,
-                          StringRef ErrorMsg, StringRef Output, Optional<int> Signal);
+                          StringRef ErrorMsg, StringRef Output,
+                          Optional<int> Signal,
+                          sys::TaskProcessInformation ProcInfo);
 
 /// \brief Emits a "skipped" message to the given stream.
 void emitSkippedMessage(raw_ostream &os, const Job &Cmd);

lib/Basic/Default/TaskQueue.inc

@@ -117,7 +117,7 @@ bool TaskQueue::execute(TaskBeganCallback Began, TaskFinishedCallback Finished,
       // a signal during execution.
       if (Signalled) {
         TaskFinishedResponse Response =
-            Signalled(PI.Pid, ErrMsg, StringRef(), StringRef(), T->Context, None);
+            Signalled(PI.Pid, ErrMsg, StringRef(), StringRef(), T->Context, None, TaskProcessInformation(PI.Pid));
         ContinueExecution = Response != TaskFinishedResponse::StopExecution;
       } else {
         // If we don't have a Signalled callback, unconditionally stop.
@@ -128,7 +128,7 @@ bool TaskQueue::execute(TaskBeganCallback Began, TaskFinishedCallback Finished,
       // finished.
       if (Finished) {
         TaskFinishedResponse Response = Finished(PI.Pid, PI.ReturnCode,
-        StringRef(), StringRef(), T->Context);
+        StringRef(), StringRef(), TaskProcessInformation(PI.Pid), T->Context);
         ContinueExecution = Response != TaskFinishedResponse::StopExecution;
       } else if (PI.ReturnCode != 0) {
         ContinueExecution = false;

lib/Basic/TaskQueue.cpp

@@ -85,8 +85,8 @@ bool DummyTaskQueue::execute(TaskQueue::TaskBeganCallback Began,
       std::string Output = "Output placeholder\n";
       std::string Errors =
           P.second->SeparateErrors ? "Error placeholder\n" : "";
-      if (Finished(P.first, 0, Output, Errors, P.second->Context) ==
-          TaskFinishedResponse::StopExecution)
+      if (Finished(P.first, 0, Output, Errors, TaskProcessInformation(Pid),
+                   P.second->Context) == TaskFinishedResponse::StopExecution)
         SubtaskFailed = true;
     }
   }