Skip to content

[LIT] Print discovered tests and percentages (#66057) #69831

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions llvm/utils/lit/lit/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@ def print_histogram(tests):

def print_results(tests, elapsed, opts):
tests_by_code = {code: [] for code in lit.Test.ResultCode.all_codes()}
total_tests = len(tests)
for test in tests:
tests_by_code[test.result.code].append(test)

Expand All @@ -321,7 +322,7 @@ def print_results(tests, elapsed, opts):
opts.shown_codes,
)

print_summary(tests_by_code, opts.quiet, elapsed)
print_summary(total_tests, tests_by_code, opts.quiet, elapsed)


def print_group(tests, code, shown_codes):
Expand All @@ -336,10 +337,11 @@ def print_group(tests, code, shown_codes):
sys.stdout.write("\n")


def print_summary(tests_by_code, quiet, elapsed):
def print_summary(total_tests, tests_by_code, quiet, elapsed):
if not quiet:
print("\nTesting Time: %.2fs" % elapsed)

print("\nTotal Discovered Tests: %s" % (total_tests))
codes = [c for c in lit.Test.ResultCode.all_codes() if not quiet or c.isFailure]
groups = [(c.label, len(tests_by_code[c])) for c in codes]
groups = [(label, count) for label, count in groups if count]
Expand All @@ -352,4 +354,4 @@ def print_summary(tests_by_code, quiet, elapsed):
for (label, count) in groups:
label = label.ljust(max_label_len)
count = str(count).rjust(max_count_len)
print(" %s: %s" % (label, count))
print(" %s: %s (%.2f%%)" % (label, count, float(count) / total_tests * 100))
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution name 'foo %{name}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK: Substitution name '%{name}:' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution name '%{}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
# CHECK: Substitution name '%{foo.bar}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK: Substitution name '%{foo' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK: Substitution name '%{foo bar}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
# CHECK: Substitution name '%{3}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution name '%{foo bar}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution name '' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@
# CHECK: Substitution's continuation is empty
# CHECK-NEXT: in DEFINE: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@
# CHECK: Substitution's continuation is empty
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{REDEFINE}}: directive on test line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') from line [[#@LINE-3]] to [[#@LINE-2]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@

# CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

# CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-2]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') from line [[#@LINE-3]] to [[#@LINE-2]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@

# CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

# CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-2]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{RUN}}:' directive (with '\') at line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{RUN}}:' directive (with '\') at line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@
# CHECK: Substitution's continuation is empty
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK-NEXT: Existing pattern: %{global:greeting}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@
# CHECK-NEXT: Existing pattern: %{foo}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK-NEXT: Existing pattern: <%{global:inside}>
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: %{global:multiple-exact}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: %{global:multiple-once-exact}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK-NEXT: Existing pattern: %{global:prefix}\((.*)\)
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK-NEXT: Existing pattern: @%{global:suffix}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: <%{global:inside}>
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
# CHECK-NEXT: Existing pattern: %{global:multiple-exact}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
# CHECK-NEXT: Existing pattern: %{global:multiple-once-exact}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: ValueError: No substitution for '%{local}' is defined before '{{REDEFINE}}:' directive at line [[#@LINE-3]]
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: %{global:prefix}\((.*)\)
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: @%{global:suffix}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

# CHECK: ValueError: Substitution whose pattern contains '%{foo}' is already defined before '{{DEFINE}}:' directive from line [[#@LINE-4]] to [[#@LINE-3]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

# CHECK: Test has no '{{RUN}}:' line

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,4 @@
; RUN: %{check}
; CHECK:# | %clang_cc1 -verify -fopenmp -fopenmp-version=51 -triple x86_64-unknown-linux-gnu -emit-llvm -o - {{.*}} | FileCheck -check-prefix=NO-SIMD {{.*}}

; CHECK: Passed: 1
; CHECK: Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,4 @@
# RUN: %{local:echo}
# CHECK:# | LOCAL: So Long World World

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@
# RUN: echo '%{lines-rel}'
# CHECK:# | [[#@LINE+1]], [[#@LINE-3]], [[#@LINE-12]]

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
2 changes: 1 addition & 1 deletion llvm/utils/lit/tests/Inputs/shtest-define/name-chars.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@
# RUN: echo '%{_foo_bar_}'
# CHECK:# | ok at [[#@LINE - 2]]

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
# CHECK-NON-RECUR:# | %{inner}
# CHECK-RECUR:# | expanded

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@
# DEFINE: %{local:echo}=echo LOCAL0: subst
# RUN: %{local:echo}

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@
# DEFINE: %{local:echo}=echo LOCAL1: subst
# RUN: %{local:echo}

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
2 changes: 1 addition & 1 deletion llvm/utils/lit/tests/Inputs/shtest-define/value-equals.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@
# RUN: echo '%{continue-equals}'
# CHECK:# | FileCheck -input-file=test.txt -implicit-check-not=foobar -check-prefixes=FOO,BAR

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# RUN: echo '%{escape}'
# CHECK:# | {{\\?}}\n {{\\?}}\g<param>

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -132,4 +132,4 @@
# RUN: %{deceptive-continue}
# CHECK:# | x=3

# CHECK:{{ *}}Passed: 1
# CHECK:{{ *}}Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
7 changes: 7 additions & 0 deletions llvm/utils/lit/tests/discovery.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@
# CHECK-BASIC-OUT: top-level-suite :: test-one
# CHECK-BASIC-OUT: top-level-suite :: test-two

# RUN: %{lit} %{inputs}/discovery \
# RUN: -v > %t.out 2> %t.err
# RUN: FileCheck --check-prefix=CHECK-PERCENTAGES-OUT < %/t.out %s
#
# CHECK-PERCENTAGES-OUT: Total Discovered Tests: {{[0-9]*}}
# CHECK-PERCENTAGES-OUT: Passed: {{[0-9]*}} {{\([0-9]*\.[0-9]*%\)}}

# Check discovery when providing the special builtin 'config_map'
# RUN: %{python} %{inputs}/config-map-discovery/driver.py \
# RUN: %{inputs}/config-map-discovery/main-config/lit.cfg \
Expand Down
9 changes: 5 additions & 4 deletions llvm/utils/lit/tests/ignore-fail.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
# CHECK-DAG: XPASS: ignore-fail :: xpass.txt

# CHECK: Testing Time:
# CHECK-NEXT: Expectedly Failed : 1
# CHECK-NEXT: Unresolved : 1
# CHECK-NEXT: Failed : 1
# CHECK-NEXT: Unexpectedly Passed: 1
# CHECK: Total Discovered Tests:
# CHECK-NEXT: Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NOT: {{.}}
5 changes: 3 additions & 2 deletions llvm/utils/lit/tests/shtest-env.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@
# CHECK-NOT: {{^[^#]}}
# CHECK: --

# CHECK: Passed: 4
# CHECK: Failed: 12
# CHECK: Total Discovered Tests: 16
# CHECK: Passed: 4 {{\([0-9]*\.[0-9]*%\)}}
# CHECK: Failed: 12 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NOT: {{.}}
5 changes: 3 additions & 2 deletions llvm/utils/lit/tests/shtest-not.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@
# CHECK: # | Error: 'not --crash' cannot call 'rm'
# CHECK: # error: command failed with exit status: {{.*}}

# CHECK: Passed: 1
# CHECK: Failed: 16
# CHECK: Total Discovered Tests: 17
# CHECK: Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK: Failed: 16 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NOT: {{.}}
Loading