[SR-755] Tool for re-symbolicating fatal stacktraces on Linux.

.pep8

@@ -1,2 +1,2 @@
 [flake8]
-filename = *.py,80+-check,backtrace-check,Benchmark_Driver,Benchmark_DTrace.in,Benchmark_GuardMalloc.in,Benchmark_RuntimeLeaksRunner.in,build-script,check-incremental,clang++,coverage-build-db,coverage-generate-data,coverage-touch-tests,gyb,ld,line-directive,mock-distcc,ns-html2rst,PathSanitizingFileCheck,recursive-lipo,rth,run-test,submit-benchmark-results,update-checkout,viewcfg
+filename = *.py,80+-check,backtrace-check,Benchmark_Driver,Benchmark_DTrace.in,Benchmark_GuardMalloc.in,Benchmark_RuntimeLeaksRunner.in,build-script,check-incremental,clang++,coverage-build-db,coverage-generate-data,coverage-touch-tests,gyb,ld,line-directive,mock-distcc,ns-html2rst,PathSanitizingFileCheck,recursive-lipo,rth,run-test,submit-benchmark-results,update-checkout,viewcfg,symbolicate-linux-fatal

stdlib/public/runtime/Errors.cpp

@@ -69,10 +69,7 @@ static bool getSymbolNameAddr(llvm::StringRef libraryName, Dl_info dlinfo,
   // need to use the hex address.
   bool hasUnavailableAddress = dlinfo.dli_sname == nullptr;
 
-  // If the address is unavailable, just use <unavailable> as the symbol
-  // name. We do not set addrOut, since addrOut will be set to our ptr address.
   if (hasUnavailableAddress) {
-    symbolName += "<unavailable>";
     return false;
   }
 
@@ -132,12 +129,21 @@ static void dumpStackTraceEntry(unsigned index, void *framePC) {
   // We do not use %p here for our pointers since the format is implementation
   // defined. This makes it logically impossible to check the output. Forcing
   // hexadecimal solves this issue.
-  static const char *backtraceEntryFormat = "%-4u %-34s 0x%0.16lx %s + %td\n";
-
-  // Then dump the backtrace.
-  fprintf(stderr, backtraceEntryFormat, index, libraryName.data(),
-          foundSymbol ? symbolAddr : uintptr_t(framePC), symbolName.c_str(),
-          ptrdiff_t(uintptr_t(framePC) - symbolAddr));
+  // If the symbol is not available, we print out <unavailable> + offset
+  // from the base address of where the image containing framePC is mapped.
+  // This gives enough info to reconstruct identical debugging target after
+  // this process terminates.
+  if (foundSymbol) {
+    static const char *backtraceEntryFormat = "%-4u %-34s 0x%0.16lx %s + %td\n";
+    fprintf(stderr, backtraceEntryFormat, index, libraryName.data(), symbolAddr,
+            symbolName.c_str(), ptrdiff_t(uintptr_t(framePC) - symbolAddr));
+  } else {
+    static const char *backtraceEntryFormat = "%-4u %-34s 0x%0.16lx "
+                                              "<unavailable> + %td\n";
+    fprintf(stderr, backtraceEntryFormat, index, libraryName.data(),
+            uintptr_t(framePC),
+            ptrdiff_t(uintptr_t(framePC) - uintptr_t(dlinfo.dli_fbase)));
+  }
 }
 
 #endif

test/Runtime/linux-fatal-backtrace.swift

@@ -0,0 +1,23 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: %target-build-swift %s -o %t/a.out
+// RUN: not --crash %t/a.out 2>&1 | PYTHONPATH=%lldb-python-path %utils/symbolicate-linux-fatal %t/a.out - | %utils/backtrace-check -u
+
+// REQUIRES: executable_test
+// REQUIRES: OS=linux-gnu
+// REQUIRES: lldb
+
+// Backtraces are not emitted when optimizations are enabled. This test can not
+// run when optimizations are enabled.
+// REQUIRES: swift_test_mode_optimize_none
+
+func funcB() {
+    fatalError("linux-fatal-backtrace");
+}
+
+func funcA() {
+    funcB();
+}
+
+print("bla")
+funcA()

test/lit.site.cfg.in

@@ -14,8 +14,6 @@ import os
 import platform
 import sys
 
-## Autogenerated by Swift configuration.
-# Do not edit!
 config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
@@ -76,6 +74,13 @@ if "@CMAKE_GENERATOR@" == "Xcode":
     config.environment['PATH'] = \
       os.path.pathsep.join((xcode_bin_dir, config.environment['PATH']))
 
+if "@LLDB_ENABLE@" == "TRUE":
+    config.available_features.add('lldb')
+    for root, dirs, files in os.walk("@LLDB_BUILD_DIR@"):
+        if root.endswith("site-packages"):
+            config.substitutions.append(('%lldb-python-path', root))
+            break
+
 # Let the main config do the real work.
 if config.test_exec_root is None:
     config.test_exec_root = os.path.dirname(os.path.realpath(__file__))

utils/backtrace-check

@@ -18,44 +18,64 @@
 # 11 libswiftCore.dylib 0x000000000dce84d0l _fatalErrorMessage(StaticString,
 # StaticString, StaticString, UInt, flags : UInt32) -> () + 444
 
+import argparse
 import re
 import sys
 
-TARGET_RE = re.compile(
-    "(?P<index>\d+) +(?P<object>\S+) +(?P<address>0x[0-9a-fA-F]{16}) "
-    "(?P<routine>[^+]+) [+] (?P<offset>\d+)")
-
-lines = sys.stdin.readlines()
-
-found_stack_trace_start = False
-found_stack_trace_entry = False
-for l in lines:
-    l = l.rstrip("\n")
-
-    # First see if we found the start of our stack trace start. If so, set the
-    # found stack trace flag and continue.
-    if l == "Current stack trace:":
-        assert(not found_stack_trace_start)
-        found_stack_trace_start = True
-        continue
-
-    # Otherwise, if we have not yet found the stack trace start, continue. We
-    # need to find the stack trace start line.
-    if not found_stack_trace_start:
-        continue
-
-    # Ok, we are in the middle of matching a stack trace entry.
-    m = TARGET_RE.match(l)
-    # If we fail to match, we have exited the stack trace entry region
-    if m is None:
-        break
-    # At this point, we know that we have some sort of match.
-    found_stack_trace_entry = True
-    print("Stack Trace Entry:")
-    print("\tIndex: '%(index)s'\n\tObject File: '%(object)s'\n\tAddress: "
-          "'%(address)s'\n\tRoutine: '%(routine)s'\n\tOffset: '%(offset)s'"
-          "\n" % m.groupdict())
-
-# Once we have processed all of the lines, make sure that we found at least one
-# stack trace entry.
-assert(found_stack_trace_entry)
+
+def main():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="""Checks that a stacktrace dump follows canonical
+formatting.""")
+    parser.add_argument(
+        "-u", "--check-unavailable", action='store_true',
+        help="Checks if any symbols were unavailable")
+    args = parser.parse_args()
+
+    TARGET_RE = re.compile(
+        "(?P<index>\d+) +(?P<object>\S+) +(?P<address>0x[0-9a-fA-F]{16}) "
+        "(?P<routine>[^+]+) [+] (?P<offset>\d+)")
+
+    lines = sys.stdin.readlines()
+
+    found_stack_trace_start = False
+    found_stack_trace_entry = False
+    for l in lines:
+        l = l.rstrip("\n")
+
+        # First see if we found the start of our stack trace start. If so, set
+        # the found stack trace flag and continue.
+        if l == "Current stack trace:":
+            assert(not found_stack_trace_start)
+            found_stack_trace_start = True
+            continue
+
+        # Otherwise, if we have not yet found the stack trace start, continue.
+        # We need to find the stack trace start line.
+        if not found_stack_trace_start:
+            continue
+
+        # Ok, we are in the middle of matching a stack trace entry.
+        m = TARGET_RE.match(l)
+        # If we fail to match, we have exited the stack trace entry region
+        if m is None:
+            break
+
+        # At this point, we know that we have some sort of match.
+        found_stack_trace_entry = True
+        print("Stack Trace Entry:")
+        print("\tIndex: '%(index)s'\n\tObject File: '%(object)s'\n\tAddress: "
+              "'%(address)s'\n\tRoutine: '%(routine)s'\n\tOffset: '%(offset)s'"
+              "\n" % m.groupdict())
+
+        # Check for unavailable symbols, if that was requested.
+        if args.check_unavailable:
+            assert("unavailable" not in m.group("routine"))
+
+    # Once we have processed all of the lines, make sure that we found at least
+    # one stack trace entry.
+    assert(found_stack_trace_entry)
+
+if __name__ == '__main__':
+    main()

utils/build-script-impl

@@ -1262,6 +1262,18 @@ if [[ ! "${SKIP_BUILD_SWIFTPM}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" swiftpm)
 fi
 
+# Checks if a given product is enabled (i.e. part of $PRODUCTS array)
+function contains_product() {
+  local current_product
+  for current_product in "${PRODUCTS[@]}"; do
+    if [[ "$current_product" == "$1" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+
 # get_host_specific_variable(host, name)
 #
 # Get the value of a host-specific variable expected to have been passed by the
@@ -2098,6 +2110,15 @@ for host in "${ALL_HOSTS[@]}"; do
                     )
                 fi
 
+                if contains_product "lldb" ; then
+                    lldb_build_dir=$(build_directory ${host} lldb)
+                    cmake_options=(
+                        "${cmake_options[@]}"
+                        -DLLDB_ENABLE:BOOL=TRUE
+                        -DLLDB_BUILD_DIR:STRING="${lldb_build_dir}"
+                    )
+                fi
+
                 build_targets=(all "${SWIFT_STDLIB_TARGETS[@]}")
                 if [[ $(true_false "${build_perf_testsuite_this_time}") == "TRUE" ]]; then
                     native_swift_tools_path="$(build_directory_bin ${LOCAL_HOST} swift)"

utils/symbolicate-linux-fatal

@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# symbolicate-linux-fatal - Symbolicate Linux stack traces -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# Symbolicates fatalError stack traces on Linux. Takes the main binary
+# and a log file containing a stack trace. Non-stacktrace lines are output
+# unmodified. Stack trace elements are analysed using reconstructed debug
+# target matching the original process in where shared libs where mapped.
+#
+# TODOs:
+# * verbose output
+# * search symbols by name for the not <unavailable> ones
+#
+# ----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import argparse
+import subprocess
+
+import lldb
+
+
+def process_ldd(lddoutput):
+    dyn_libs = {}
+    for line in lddoutput.splitlines():
+        ldd_tokens = line.split()
+        if len(ldd_tokens) >= 3:
+            dyn_libs[ldd_tokens[0]] = ldd_tokens[2]
+    return dyn_libs
+
+
+def create_lldb_target(binary, memmap):
+    lldb_debugger = lldb.SBDebugger.Create()
+    lldb_target = lldb_debugger.CreateTargetWithFileAndArch(
+        binary, lldb.LLDB_ARCH_DEFAULT)
+    module = lldb_target.GetModuleAtIndex(0)
+    # lldb seems to treat main binary differently, slide offset must be zero
+    lldb_target.SetModuleLoadAddress(module, 0)
+    for dynlib_path in memmap:
+        if binary not in dynlib_path:
+            module = lldb_target.AddModule(
+                dynlib_path, lldb.LLDB_ARCH_DEFAULT, None, None)
+            lldb_target.SetModuleLoadAddress(module, memmap[dynlib_path])
+    return lldb_target
+
+
+def process_stack(binary, dyn_libs, stack):
+    if len(stack) == 0:
+        return
+    memmap = {}
+    full_stack = []
+    for line in stack:
+        stack_tokens = line.split()
+        dynlib_fname = stack_tokens[1]
+        if dynlib_fname in dyn_libs:
+            dynlib_path = dyn_libs[dynlib_fname]
+        elif dynlib_fname in binary:
+            dynlib_path = binary
+        else:
+            dynlib_path = None
+
+        if "<unavailable>" in stack_tokens[3]:
+            framePC = int(stack_tokens[2], 16)
+            symbol_offset = int(stack_tokens[-1], 10)
+            dynlib_baseaddr = framePC - symbol_offset
+            if dynlib_path in memmap:
+                if memmap[dynlib_path] != dynlib_baseaddr:
+                    error_msg = "Mismatched base address for: {0:s}, " \
+                                "had: {1:x}, now got {2:x}"
+                    error_msg = error_msg.format(
+                        dynlib_path, memmap[dynlib_path], dynlib_baseaddr)
+                    raise Exception(error_msg)
+            else:
+                memmap[dynlib_path] = dynlib_baseaddr
+        else:
+            framePC = int(stack_tokens[2], 16) + int(stack_tokens[-1], 10)
+        full_stack.append(
+            {"line": line, "framePC": framePC, "dynlib_fname": dynlib_fname})
+
+    lldb_target = create_lldb_target(binary, memmap)
+    frame_idx = 0
+    for frame in full_stack:
+        use_orig_line = True
+        frame_addr = frame["framePC"]
+        dynlib_fname = frame["dynlib_fname"]
+        so_addr = lldb_target.ResolveLoadAddress(frame_addr-1)
+        sym_ctx = so_addr.GetSymbolContext(lldb.eSymbolContextEverything)
+        frame_fragment = "{0: <4d} {1:20s} 0x{2:016x}".format(
+            frame_idx, dynlib_fname, frame_addr)
+        symbol = sym_ctx.GetSymbol()
+        if symbol.IsValid():
+            symbol_base = symbol.GetStartAddress().GetLoadAddress(lldb_target)
+            symbol_fragment = "{0:s} + {1:d}".format(
+                symbol.GetName(), frame_addr - symbol_base)
+            use_orig_line = False
+        else:
+            symbol_fragment = "<unavailable>"
+        line_entry = sym_ctx.GetLineEntry()
+        if line_entry.IsValid():
+            line_fragment = "at {0:s}:{1:d}".format(
+                line_entry.GetFileSpec().GetFilename(), line_entry.GetLine())
+        else:
+            line_fragment = ""
+        if use_orig_line:
+            print(frame["line"].rstrip())
+        else:
+            print("{0:s}    {1:s} {2:s}".format(
+                frame_fragment, symbol_fragment, line_fragment))
+        frame_idx = frame_idx + 1
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="""Symbolicates stack traces in Linux log files.""")
+    parser.add_argument(
+        "binary", help="Executable which produced the log file")
+    parser.add_argument(
+        "log", type=argparse.FileType("rU"),
+        help="Log file containing the stack trace to symbolicate")
+    args = parser.parse_args()
+
+    binary = args.binary
+
+    lddoutput = subprocess.check_output(
+        ['ldd', binary], stderr=subprocess.STDOUT)
+    dyn_libs = process_ldd(lddoutput)
+
+    instack = False
+    stackidx = 0
+    stack = []
+    for line in args.log:
+        if instack and line.startswith(str(stackidx)):
+            stack.append(line)
+            stackidx = stackidx + 1
+        else:
+            instack = False
+            stackidx = 0
+            process_stack(binary, dyn_libs, stack)
+            stack = []
+            print(line.rstrip())
+        if line.startswith("Current stack trace:"):
+            instack = True
+    process_stack(binary, dyn_libs, stack)
+
+
+if __name__ == '__main__':
+    main()

validation-test/lit.site.cfg.in

@@ -13,8 +13,6 @@
 import sys
 import platform
 
-## Autogenerated by Swift configuration.
-# Do not edit!
 config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"