Fix symbolicate linux fatal for streaming stacktraces 2

utils/symbolicate-linux-fatal

@@ -25,8 +25,10 @@
 from __future__ import print_function
 
 import argparse
+import datetime
 import os
 import subprocess
+import sys
 
 try:
     import lldb
@@ -36,10 +38,17 @@ except ImportError:
     if swift_exec is not None:
         site_packages = os.path.join(os.path.dirname(swift_exec),
                                      '../lib/python2.7/site-packages/')
-        import sys
         sys.path.append(site_packages)
         import lldb
 
+lldb_target = None
+known_memmap = {}
+
+
+def print_with_flush(buff):
+    print(buff)
+    sys.stdout.flush()
+
 
 def process_ldd(lddoutput):
     dyn_libs = {}
@@ -53,28 +62,18 @@ def process_ldd(lddoutput):
     return dyn_libs
 
 
-def create_lldb_target(binary, memmap):
-    lldb_debugger = lldb.SBDebugger.Create()
-    lldb_target = lldb_debugger.CreateTarget(binary)
-    module = lldb_target.GetModuleAtIndex(0)
+def setup_lldb_target(binary, memmap):
+    global lldb_target
+    if not lldb_target:
+        lldb_debugger = lldb.SBDebugger.Create()
+        lldb_target = lldb_debugger.CreateTarget(binary)
+        module = lldb_target.GetModuleAtIndex(0)
     for dynlib_path in memmap:
         module = lldb_target.AddModule(
             dynlib_path, lldb.LLDB_ARCH_DEFAULT, None, None)
         text_section = module.FindSection(".text")
         slide = text_section.GetFileAddress() - text_section.GetFileOffset()
         lldb_target.SetModuleLoadAddress(module, memmap[dynlib_path] - slide)
-    return lldb_target
-
-
-def add_lldb_target_modules(lldb_target, memmap):
-    for dynlib_path in memmap:
-        module = lldb_target.AddModule(
-            dynlib_path, lldb.LLDB_ARCH_DEFAULT, None, None)
-        lldb_target.SetModuleLoadAddress(module, memmap[dynlib_path])
-
-
-lldb_target = None
-known_memmap = {}
 
 
 def check_base_address(dynlib_path, dynlib_baseaddr, memmap):
@@ -116,11 +115,12 @@ def symbolicate_one(frame_addr, frame_idx, dynlib_fname):
            frame_fragment, symbol_fragment, line_fragment)
 
 
-def process_stack(binary, dyn_libs, stack):
+def get_processed_stack(binary, dyn_libs, stack):
     global lldb_target
     global known_memmap
+    processed_stack = []
     if len(stack) == 0:
-        return
+        return processed_stack
     memmap = {}
     full_stack = []
     for line in stack:
@@ -150,20 +150,68 @@ def process_stack(binary, dyn_libs, stack):
         full_stack.append(
             {"line": line, "framePC": framePC, "dynlib_fname": dynlib_fname})
 
-    if lldb_target is None:
-        lldb_target = create_lldb_target(binary, memmap)
-    else:
-        add_lldb_target_modules(lldb_target, memmap)
-    frame_idx = 0
-    for frame in full_stack:
+    setup_lldb_target(binary, memmap)
+
+    for frame_idx, frame in enumerate(full_stack):
         frame_addr = frame["framePC"]
         dynlib_fname = frame["dynlib_fname"]
         try:
             sym_line = symbolicate_one(frame_addr, frame_idx, dynlib_fname)
-            print(sym_line)
+            processed_stack.append(sym_line)
         except Exception:
-            print(frame["line"].rstrip())
-        frame_idx = frame_idx + 1
+            processed_stack.append(frame["line"].rstrip())
+
+    return processed_stack
+
+
+def is_fatal_error(line):
+    return line.startswith("Fatal error:")
+
+
+def is_stack_trace_header(line):
+    return line.startswith("Current stack trace:")
+
+
+def should_print_previous_line(current_line, previous_line):
+    return is_fatal_error(previous_line) and \
+        not is_stack_trace_header(current_line)
+
+
+def should_print_current_line(current_line, previous_line):
+    return (not is_fatal_error(current_line) and
+            not is_stack_trace_header(current_line)) or \
+           (is_stack_trace_header(current_line) and
+            not is_fatal_error(previous_line))
+
+
+def fatal_error_with_stack_trace_found(current_line, previous_line):
+    return is_stack_trace_header(current_line) and \
+        is_fatal_error(previous_line)
+
+
+def print_stack(fatal_error_header,
+                fatal_error_stack_trace_header,
+                fatal_log_format,
+                processed_stack):
+    if not fatal_error_header:
+        for line in processed_stack:
+            print_with_flush(line)
+    else:
+        # fatal error with a stack trace
+        stack_str = fatal_error_header + fatal_error_stack_trace_header + \
+            '\n'.join(processed_stack)
+        formatted_output = fatal_log_format
+
+        if "%t" in formatted_output:
+            current_time = datetime.datetime.now()
+            time_in_iso_format = \
+                current_time.strftime('%Y-%m-%dT%H:%M:%S,%f%z')
+            formatted_output = \
+                formatted_output.replace("%t", time_in_iso_format)
+        if "%m" in formatted_output:
+            formatted_output = formatted_output.replace("%m", stack_str)
+
+        print_with_flush(formatted_output)
 
 
 def main():
@@ -175,34 +223,69 @@ def main():
     parser.add_argument(
         "log", nargs='?', type=argparse.FileType("rU"), default="-",
         help="Log file for symbolication. Defaults to stdin.")
+    parser.add_argument(
+        "--fatal-log-format", default="%m",
+        help="Format for logging fatal errors. Variable %%t will be "
+             "replaced with current time in ISO 8601 format, variable "
+             "%%m will be replaced with the error message with a full "
+             "stack trace.")
     args = parser.parse_args()
 
     binary = args.binary
+    fatal_log_format = args.fatal_log_format
 
     lddoutput = subprocess.check_output(
         ['ldd', binary], stderr=subprocess.STDOUT)
     dyn_libs = process_ldd(lddoutput)
 
     instack = False
+    previous_line = ""
     stackidx = 0
     stack = []
+    fatal_error_header = ""
+    fatal_error_stack_trace_header = ""
 
     while True:
-        line = args.log.readline()
-        if not line:
+        current_line = args.log.readline()
+        if not current_line:
             break
-        if instack and line.startswith(str(stackidx)):
-            stack.append(line)
+        if instack and current_line.startswith(str(stackidx)):
+            stack.append(current_line)
             stackidx = stackidx + 1
         else:
+            processed_stack = get_processed_stack(binary, dyn_libs, stack)
+            print_stack(fatal_error_header,
+                        fatal_error_stack_trace_header,
+                        fatal_log_format,
+                        processed_stack)
+
             instack = False
             stackidx = 0
-            process_stack(binary, dyn_libs, stack)
             stack = []
-            print(line.rstrip())
-        if line.startswith("Current stack trace:"):
-            instack = True
-    process_stack(binary, dyn_libs, stack)
+            fatal_error_header = ""
+            fatal_error_stack_trace_header = ""
+
+            if is_stack_trace_header(current_line):
+                instack = True
+
+            if should_print_previous_line(current_line, previous_line):
+                print_with_flush(previous_line.rstrip())
+
+            if should_print_current_line(current_line, previous_line):
+                print_with_flush(current_line.rstrip())
+
+            if fatal_error_with_stack_trace_found(current_line, previous_line):
+                fatal_error_header = previous_line
+                fatal_error_stack_trace_header = current_line
+
+        previous_line = current_line
+    if is_fatal_error(previous_line):
+        print_with_flush(previous_line.rstrip())
+    processed_stack = get_processed_stack(binary, dyn_libs, stack)
+    print_stack(fatal_error_header,
+                fatal_error_stack_trace_header,
+                fatal_log_format,
+                processed_stack)
 
 
 if __name__ == '__main__':