Skip to content

Commit f1a4dd7

Browse files
authored
Merge pull request #4479 from gmilos/SR-755-linux-fatal-stacktrace-symbolication
[SR-755] Tool for re-symbolicating fatal stacktraces on Linux.
2 parents 2942b85 + f96b655 commit f1a4dd7

File tree

8 files changed

+283
-52
lines changed

8 files changed

+283
-52
lines changed

.pep8

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
[flake8]
2-
filename = *.py,80+-check,backtrace-check,Benchmark_Driver,Benchmark_DTrace.in,Benchmark_GuardMalloc.in,Benchmark_RuntimeLeaksRunner.in,build-script,check-incremental,clang++,coverage-build-db,coverage-generate-data,coverage-touch-tests,gyb,ld,line-directive,mock-distcc,ns-html2rst,PathSanitizingFileCheck,recursive-lipo,rth,run-test,submit-benchmark-results,update-checkout,viewcfg
2+
filename = *.py,80+-check,backtrace-check,Benchmark_Driver,Benchmark_DTrace.in,Benchmark_GuardMalloc.in,Benchmark_RuntimeLeaksRunner.in,build-script,check-incremental,clang++,coverage-build-db,coverage-generate-data,coverage-touch-tests,gyb,ld,line-directive,mock-distcc,ns-html2rst,PathSanitizingFileCheck,recursive-lipo,rth,run-test,submit-benchmark-results,update-checkout,viewcfg,symbolicate-linux-fatal

stdlib/public/runtime/Errors.cpp

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,7 @@ static bool getSymbolNameAddr(llvm::StringRef libraryName, Dl_info dlinfo,
6969
// need to use the hex address.
7070
bool hasUnavailableAddress = dlinfo.dli_sname == nullptr;
7171

72-
// If the address is unavailable, just use <unavailable> as the symbol
73-
// name. We do not set addrOut, since addrOut will be set to our ptr address.
7472
if (hasUnavailableAddress) {
75-
symbolName += "<unavailable>";
7673
return false;
7774
}
7875

@@ -132,12 +129,21 @@ static void dumpStackTraceEntry(unsigned index, void *framePC) {
132129
// We do not use %p here for our pointers since the format is implementation
133130
// defined. This makes it logically impossible to check the output. Forcing
134131
// hexadecimal solves this issue.
135-
static const char *backtraceEntryFormat = "%-4u %-34s 0x%0.16lx %s + %td\n";
136-
137-
// Then dump the backtrace.
138-
fprintf(stderr, backtraceEntryFormat, index, libraryName.data(),
139-
foundSymbol ? symbolAddr : uintptr_t(framePC), symbolName.c_str(),
140-
ptrdiff_t(uintptr_t(framePC) - symbolAddr));
132+
// If the symbol is not available, we print out <unavailable> + offset
133+
// from the base address of where the image containing framePC is mapped.
134+
// This gives enough info to reconstruct identical debugging target after
135+
// this process terminates.
136+
if (foundSymbol) {
137+
static const char *backtraceEntryFormat = "%-4u %-34s 0x%0.16lx %s + %td\n";
138+
fprintf(stderr, backtraceEntryFormat, index, libraryName.data(), symbolAddr,
139+
symbolName.c_str(), ptrdiff_t(uintptr_t(framePC) - symbolAddr));
140+
} else {
141+
static const char *backtraceEntryFormat = "%-4u %-34s 0x%0.16lx "
142+
"<unavailable> + %td\n";
143+
fprintf(stderr, backtraceEntryFormat, index, libraryName.data(),
144+
uintptr_t(framePC),
145+
ptrdiff_t(uintptr_t(framePC) - uintptr_t(dlinfo.dli_fbase)));
146+
}
141147
}
142148

143149
#endif
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// RUN: rm -rf %t
2+
// RUN: mkdir -p %t
3+
// RUN: %target-build-swift %s -o %t/a.out
4+
// RUN: not --crash %t/a.out 2>&1 | PYTHONPATH=%lldb-python-path %utils/symbolicate-linux-fatal %t/a.out - | %utils/backtrace-check -u
5+
6+
// REQUIRES: executable_test
7+
// REQUIRES: OS=linux-gnu
8+
// REQUIRES: lldb
9+
10+
// Backtraces are not emitted when optimizations are enabled. This test can not
11+
// run when optimizations are enabled.
12+
// REQUIRES: swift_test_mode_optimize_none
13+
14+
func funcB() {
15+
fatalError("linux-fatal-backtrace");
16+
}
17+
18+
func funcA() {
19+
funcB();
20+
}
21+
22+
print("bla")
23+
funcA()

test/lit.site.cfg.in

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,6 @@ import os
1414
import platform
1515
import sys
1616

17-
## Autogenerated by Swift configuration.
18-
# Do not edit!
1917
config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
2018
config.llvm_obj_root = "@LLVM_BINARY_DIR@"
2119
config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
@@ -76,6 +74,13 @@ if "@CMAKE_GENERATOR@" == "Xcode":
7674
config.environment['PATH'] = \
7775
os.path.pathsep.join((xcode_bin_dir, config.environment['PATH']))
7876

77+
if "@LLDB_ENABLE@" == "TRUE":
78+
config.available_features.add('lldb')
79+
for root, dirs, files in os.walk("@LLDB_BUILD_DIR@"):
80+
if root.endswith("site-packages"):
81+
config.substitutions.append(('%lldb-python-path', root))
82+
break
83+
7984
# Let the main config do the real work.
8085
if config.test_exec_root is None:
8186
config.test_exec_root = os.path.dirname(os.path.realpath(__file__))

utils/backtrace-check

Lines changed: 58 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -18,44 +18,64 @@
1818
# 11 libswiftCore.dylib 0x000000000dce84d0l _fatalErrorMessage(StaticString,
1919
# StaticString, StaticString, UInt, flags : UInt32) -> () + 444
2020

21+
import argparse
2122
import re
2223
import sys
2324

24-
TARGET_RE = re.compile(
25-
"(?P<index>\d+) +(?P<object>\S+) +(?P<address>0x[0-9a-fA-F]{16}) "
26-
"(?P<routine>[^+]+) [+] (?P<offset>\d+)")
27-
28-
lines = sys.stdin.readlines()
29-
30-
found_stack_trace_start = False
31-
found_stack_trace_entry = False
32-
for l in lines:
33-
l = l.rstrip("\n")
34-
35-
# First see if we found the start of our stack trace start. If so, set the
36-
# found stack trace flag and continue.
37-
if l == "Current stack trace:":
38-
assert(not found_stack_trace_start)
39-
found_stack_trace_start = True
40-
continue
41-
42-
# Otherwise, if we have not yet found the stack trace start, continue. We
43-
# need to find the stack trace start line.
44-
if not found_stack_trace_start:
45-
continue
46-
47-
# Ok, we are in the middle of matching a stack trace entry.
48-
m = TARGET_RE.match(l)
49-
# If we fail to match, we have exited the stack trace entry region
50-
if m is None:
51-
break
52-
# At this point, we know that we have some sort of match.
53-
found_stack_trace_entry = True
54-
print("Stack Trace Entry:")
55-
print("\tIndex: '%(index)s'\n\tObject File: '%(object)s'\n\tAddress: "
56-
"'%(address)s'\n\tRoutine: '%(routine)s'\n\tOffset: '%(offset)s'"
57-
"\n" % m.groupdict())
58-
59-
# Once we have processed all of the lines, make sure that we found at least one
60-
# stack trace entry.
61-
assert(found_stack_trace_entry)
25+
26+
def main():
27+
parser = argparse.ArgumentParser(
28+
formatter_class=argparse.RawDescriptionHelpFormatter,
29+
description="""Checks that a stacktrace dump follows canonical
30+
formatting.""")
31+
parser.add_argument(
32+
"-u", "--check-unavailable", action='store_true',
33+
help="Checks if any symbols were unavailable")
34+
args = parser.parse_args()
35+
36+
TARGET_RE = re.compile(
37+
"(?P<index>\d+) +(?P<object>\S+) +(?P<address>0x[0-9a-fA-F]{16}) "
38+
"(?P<routine>[^+]+) [+] (?P<offset>\d+)")
39+
40+
lines = sys.stdin.readlines()
41+
42+
found_stack_trace_start = False
43+
found_stack_trace_entry = False
44+
for l in lines:
45+
l = l.rstrip("\n")
46+
47+
# First see if we found the start of our stack trace start. If so, set
48+
# the found stack trace flag and continue.
49+
if l == "Current stack trace:":
50+
assert(not found_stack_trace_start)
51+
found_stack_trace_start = True
52+
continue
53+
54+
# Otherwise, if we have not yet found the stack trace start, continue.
55+
# We need to find the stack trace start line.
56+
if not found_stack_trace_start:
57+
continue
58+
59+
# Ok, we are in the middle of matching a stack trace entry.
60+
m = TARGET_RE.match(l)
61+
# If we fail to match, we have exited the stack trace entry region
62+
if m is None:
63+
break
64+
65+
# At this point, we know that we have some sort of match.
66+
found_stack_trace_entry = True
67+
print("Stack Trace Entry:")
68+
print("\tIndex: '%(index)s'\n\tObject File: '%(object)s'\n\tAddress: "
69+
"'%(address)s'\n\tRoutine: '%(routine)s'\n\tOffset: '%(offset)s'"
70+
"\n" % m.groupdict())
71+
72+
# Check for unavailable symbols, if that was requested.
73+
if args.check_unavailable:
74+
assert("unavailable" not in m.group("routine"))
75+
76+
# Once we have processed all of the lines, make sure that we found at least
77+
# one stack trace entry.
78+
assert(found_stack_trace_entry)
79+
80+
if __name__ == '__main__':
81+
main()

utils/build-script-impl

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1262,6 +1262,18 @@ if [[ ! "${SKIP_BUILD_SWIFTPM}" ]] ; then
12621262
PRODUCTS=("${PRODUCTS[@]}" swiftpm)
12631263
fi
12641264

1265+
# Checks if a given product is enabled (i.e. part of $PRODUCTS array)
1266+
function contains_product() {
1267+
local current_product
1268+
for current_product in "${PRODUCTS[@]}"; do
1269+
if [[ "$current_product" == "$1" ]]; then
1270+
return 0
1271+
fi
1272+
done
1273+
return 1
1274+
}
1275+
1276+
12651277
# get_host_specific_variable(host, name)
12661278
#
12671279
# Get the value of a host-specific variable expected to have been passed by the
@@ -2098,6 +2110,15 @@ for host in "${ALL_HOSTS[@]}"; do
20982110
)
20992111
fi
21002112

2113+
if contains_product "lldb" ; then
2114+
lldb_build_dir=$(build_directory ${host} lldb)
2115+
cmake_options=(
2116+
"${cmake_options[@]}"
2117+
-DLLDB_ENABLE:BOOL=TRUE
2118+
-DLLDB_BUILD_DIR:STRING="${lldb_build_dir}"
2119+
)
2120+
fi
2121+
21012122
build_targets=(all "${SWIFT_STDLIB_TARGETS[@]}")
21022123
if [[ $(true_false "${build_perf_testsuite_this_time}") == "TRUE" ]]; then
21032124
native_swift_tools_path="$(build_directory_bin ${LOCAL_HOST} swift)"

utils/symbolicate-linux-fatal

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env python
2+
# symbolicate-linux-fatal - Symbolicate Linux stack traces -*- python -*-
3+
#
4+
# This source file is part of the Swift.org open source project
5+
#
6+
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
7+
# Licensed under Apache License v2.0 with Runtime Library Exception
8+
#
9+
# See http://swift.org/LICENSE.txt for license information
10+
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11+
#
12+
# ----------------------------------------------------------------------------
13+
#
14+
# Symbolicates fatalError stack traces on Linux. Takes the main binary
15+
# and a log file containing a stack trace. Non-stacktrace lines are output
16+
# unmodified. Stack trace elements are analysed using reconstructed debug
17+
# target matching the original process in where shared libs where mapped.
18+
#
19+
# TODOs:
20+
# * verbose output
21+
# * search symbols by name for the not <unavailable> ones
22+
#
23+
# ----------------------------------------------------------------------------
24+
25+
from __future__ import print_function
26+
27+
import argparse
28+
import subprocess
29+
30+
import lldb
31+
32+
33+
def process_ldd(lddoutput):
34+
dyn_libs = {}
35+
for line in lddoutput.splitlines():
36+
ldd_tokens = line.split()
37+
if len(ldd_tokens) >= 3:
38+
dyn_libs[ldd_tokens[0]] = ldd_tokens[2]
39+
return dyn_libs
40+
41+
42+
def create_lldb_target(binary, memmap):
43+
lldb_debugger = lldb.SBDebugger.Create()
44+
lldb_target = lldb_debugger.CreateTargetWithFileAndArch(
45+
binary, lldb.LLDB_ARCH_DEFAULT)
46+
module = lldb_target.GetModuleAtIndex(0)
47+
# lldb seems to treat main binary differently, slide offset must be zero
48+
lldb_target.SetModuleLoadAddress(module, 0)
49+
for dynlib_path in memmap:
50+
if binary not in dynlib_path:
51+
module = lldb_target.AddModule(
52+
dynlib_path, lldb.LLDB_ARCH_DEFAULT, None, None)
53+
lldb_target.SetModuleLoadAddress(module, memmap[dynlib_path])
54+
return lldb_target
55+
56+
57+
def process_stack(binary, dyn_libs, stack):
58+
if len(stack) == 0:
59+
return
60+
memmap = {}
61+
full_stack = []
62+
for line in stack:
63+
stack_tokens = line.split()
64+
dynlib_fname = stack_tokens[1]
65+
if dynlib_fname in dyn_libs:
66+
dynlib_path = dyn_libs[dynlib_fname]
67+
elif dynlib_fname in binary:
68+
dynlib_path = binary
69+
else:
70+
dynlib_path = None
71+
72+
if "<unavailable>" in stack_tokens[3]:
73+
framePC = int(stack_tokens[2], 16)
74+
symbol_offset = int(stack_tokens[-1], 10)
75+
dynlib_baseaddr = framePC - symbol_offset
76+
if dynlib_path in memmap:
77+
if memmap[dynlib_path] != dynlib_baseaddr:
78+
error_msg = "Mismatched base address for: {0:s}, " \
79+
"had: {1:x}, now got {2:x}"
80+
error_msg = error_msg.format(
81+
dynlib_path, memmap[dynlib_path], dynlib_baseaddr)
82+
raise Exception(error_msg)
83+
else:
84+
memmap[dynlib_path] = dynlib_baseaddr
85+
else:
86+
framePC = int(stack_tokens[2], 16) + int(stack_tokens[-1], 10)
87+
full_stack.append(
88+
{"line": line, "framePC": framePC, "dynlib_fname": dynlib_fname})
89+
90+
lldb_target = create_lldb_target(binary, memmap)
91+
frame_idx = 0
92+
for frame in full_stack:
93+
use_orig_line = True
94+
frame_addr = frame["framePC"]
95+
dynlib_fname = frame["dynlib_fname"]
96+
so_addr = lldb_target.ResolveLoadAddress(frame_addr-1)
97+
sym_ctx = so_addr.GetSymbolContext(lldb.eSymbolContextEverything)
98+
frame_fragment = "{0: <4d} {1:20s} 0x{2:016x}".format(
99+
frame_idx, dynlib_fname, frame_addr)
100+
symbol = sym_ctx.GetSymbol()
101+
if symbol.IsValid():
102+
symbol_base = symbol.GetStartAddress().GetLoadAddress(lldb_target)
103+
symbol_fragment = "{0:s} + {1:d}".format(
104+
symbol.GetName(), frame_addr - symbol_base)
105+
use_orig_line = False
106+
else:
107+
symbol_fragment = "<unavailable>"
108+
line_entry = sym_ctx.GetLineEntry()
109+
if line_entry.IsValid():
110+
line_fragment = "at {0:s}:{1:d}".format(
111+
line_entry.GetFileSpec().GetFilename(), line_entry.GetLine())
112+
else:
113+
line_fragment = ""
114+
if use_orig_line:
115+
print(frame["line"].rstrip())
116+
else:
117+
print("{0:s} {1:s} {2:s}".format(
118+
frame_fragment, symbol_fragment, line_fragment))
119+
frame_idx = frame_idx + 1
120+
121+
122+
def main():
123+
parser = argparse.ArgumentParser(
124+
formatter_class=argparse.RawDescriptionHelpFormatter,
125+
description="""Symbolicates stack traces in Linux log files.""")
126+
parser.add_argument(
127+
"binary", help="Executable which produced the log file")
128+
parser.add_argument(
129+
"log", type=argparse.FileType("rU"),
130+
help="Log file containing the stack trace to symbolicate")
131+
args = parser.parse_args()
132+
133+
binary = args.binary
134+
135+
lddoutput = subprocess.check_output(
136+
['ldd', binary], stderr=subprocess.STDOUT)
137+
dyn_libs = process_ldd(lddoutput)
138+
139+
instack = False
140+
stackidx = 0
141+
stack = []
142+
for line in args.log:
143+
if instack and line.startswith(str(stackidx)):
144+
stack.append(line)
145+
stackidx = stackidx + 1
146+
else:
147+
instack = False
148+
stackidx = 0
149+
process_stack(binary, dyn_libs, stack)
150+
stack = []
151+
print(line.rstrip())
152+
if line.startswith("Current stack trace:"):
153+
instack = True
154+
process_stack(binary, dyn_libs, stack)
155+
156+
157+
if __name__ == '__main__':
158+
main()

validation-test/lit.site.cfg.in

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@
1313
import sys
1414
import platform
1515

16-
## Autogenerated by Swift configuration.
17-
# Do not edit!
1816
config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
1917
config.llvm_obj_root = "@LLVM_BINARY_DIR@"
2018
config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"

0 commit comments

Comments
 (0)