Skip to content

Commit 99d6e05

Browse files
author
Julian Lettner
committed
[lit] Improve naming of test result categories
Improve consistency when printing test results: Previously we were using different labels for group names (the header for the list of, e.g., failing tests) and summary count lines. For example, "Failing Tests"/"Unexpected Failures". This commit changes lit to label things consistently. Improve wording of labels: When talking about individual test results, the first word in "Unexpected Failures", "Expected Passes", and "Individual Timeouts" is superfluous. Some labels contain the word "Tests" and some don't. Let's simplify the names. Before: ``` Failing Tests (1): ... Expected Passes : 3 Unexpected Failures: 1 ``` After: ``` Failed Tests (1): ... Passed: 3 Failed: 1 ``` Reviewed By: ldionne Differential Revision: https://reviews.llvm.org/D77708
1 parent 9bcef27 commit 99d6e05

20 files changed

+71
-72
lines changed

‎clang/www/hacking.html

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -264,12 +264,12 @@ <h3 id="testingCommands">Testing on the Command Line</h3>
264264
-- Testing: Testing: 2534 tests, 4 threads --
265265
Testing: 0 .. 10.. 20.. 30.. 40.. 50.. 60.. 70.. 80.. 90..
266266
Testing Time: 81.52s
267-
Expected Passes : 2503
268-
Expected Failures : 28
269-
Unsupported Tests : 3
267+
Passed : 2503
268+
Expectedly Failed: 28
269+
Unsupported : 3
270270
</pre>
271271

272-
<p>The statistic, "Unexpected Failures" (not shown if all tests pass), is the important one.</p>
272+
<p>The statistic, "Failed" (not shown if all tests pass), is the important one.</p>
273273

274274
<!--=====================================================================-->
275275
<h2 id="patches">Creating Patch Files</h2>

‎llvm/utils/lit/lit/main.py

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -265,34 +265,33 @@ def print_histogram(tests):
265265

266266
def add_result_category(result_code, label):
267267
assert isinstance(result_code, lit.Test.ResultCode)
268-
category = (result_code, "%s Tests" % label, label)
268+
category = (result_code, label)
269269
result_codes.append(category)
270270

271271

272-
# Status code, summary label, group label
273272
result_codes = [
274273
# Passes
275-
(lit.Test.EXCLUDED, 'Excluded Tests', 'Excluded'),
276-
(lit.Test.SKIPPED, 'Skipped Tests', 'Skipped'),
277-
(lit.Test.UNSUPPORTED, 'Unsupported Tests', 'Unsupported'),
278-
(lit.Test.PASS, 'Expected Passes', ''),
279-
(lit.Test.FLAKYPASS, 'Passes With Retry', ''),
280-
(lit.Test.XFAIL, 'Expected Failures', 'Expected Failing'),
274+
(lit.Test.EXCLUDED, 'Excluded'),
275+
(lit.Test.SKIPPED, 'Skipped'),
276+
(lit.Test.UNSUPPORTED, 'Unsupported'),
277+
(lit.Test.PASS, 'Passed'),
278+
(lit.Test.FLAKYPASS, 'Passed With Retry'),
279+
(lit.Test.XFAIL, 'Expectedly Failed'),
281280
# Failures
282-
(lit.Test.UNRESOLVED, 'Unresolved Tests', 'Unresolved'),
283-
(lit.Test.TIMEOUT, 'Individual Timeouts', 'Timed Out'),
284-
(lit.Test.FAIL, 'Unexpected Failures', 'Failing'),
285-
(lit.Test.XPASS, 'Unexpected Passes', 'Unexpected Passing')
281+
(lit.Test.UNRESOLVED, 'Unresolved'),
282+
(lit.Test.TIMEOUT, 'Timed Out'),
283+
(lit.Test.FAIL, 'Failed'),
284+
(lit.Test.XPASS, 'Unexpectedly Passed')
286285
]
287286

288287

289288
def print_results(tests, elapsed, opts):
290-
tests_by_code = {code: [] for (code, _, _) in result_codes}
289+
tests_by_code = {code: [] for code, _ in result_codes}
291290
for test in tests:
292291
tests_by_code[test.result.code].append(test)
293292

294-
for (code, _, group_label) in result_codes:
295-
print_group(code, group_label, tests_by_code[code], opts)
293+
for (code, label) in result_codes:
294+
print_group(code, label, tests_by_code[code], opts)
296295

297296
print_summary(tests_by_code, opts.quiet, elapsed)
298297

@@ -318,7 +317,7 @@ def print_summary(tests_by_code, quiet, elapsed):
318317
print('\nTesting Time: %.2fs' % elapsed)
319318

320319
codes = [c for c in result_codes if not quiet or c.isFailure]
321-
groups = [(label, len(tests_by_code[code])) for code, label, _ in codes]
320+
groups = [(label, len(tests_by_code[code])) for code, label in codes]
322321
groups = [(label, count) for label, count in groups if count]
323322
if not groups:
324323
return

‎llvm/utils/lit/tests/allow-retries.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,18 @@
55
#
66
# RUN: rm -f %t.counter
77
# RUN: %{lit} -j 1 %{inputs}/allow-retries/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s
8-
# CHECK-TEST1: Passes With Retry: 1
8+
# CHECK-TEST1: Passed With Retry: 1
99

1010
# Test that a per-file ALLOW_RETRIES overwrites the config-wide test_retry_attempts property, if any.
1111
#
1212
# RUN: rm -f %t.counter
1313
# RUN: %{lit} -j 1 %{inputs}/allow-retries/succeeds-within-limit.py -Dtest_retry_attempts=2 -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST2 %s
14-
# CHECK-TEST2: Passes With Retry: 1
14+
# CHECK-TEST2: Passed With Retry: 1
1515

1616
# This test does not succeed within the allowed retry limit
1717
#
1818
# RUN: not %{lit} -j 1 %{inputs}/allow-retries/does-not-succeed-within-limit.py | FileCheck --check-prefix=CHECK-TEST3 %s
19-
# CHECK-TEST3: Failing Tests (1):
19+
# CHECK-TEST3: Failed Tests (1):
2020
# CHECK-TEST3: allow-retries :: does-not-succeed-within-limit.py
2121

2222
# This test should be UNRESOLVED since it has more than one ALLOW_RETRIES
@@ -38,4 +38,4 @@
3838
#
3939
# RUN: rm -f %t.counter
4040
# RUN: %{lit} -j 1 %{inputs}/test_retry_attempts/test.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST6 %s
41-
# CHECK-TEST6: Passes With Retry: 1
41+
# CHECK-TEST6: Passed With Retry: 1

‎llvm/utils/lit/tests/custom-result-category.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66
# CHECK: CUSTOM_PASS: custom-result-category :: test1.txt
77
# CHECK: CUSTOM_FAILURE: custom-result-category :: test2.txt
88

9-
# TODO(yln): Passing tests shouldn't be printed by default.
9+
# TODO(yln): Passed tests shouldn't be printed by default.
1010
# CHECK: My Passed Tests (1)
1111
# CHECK: My Failed Tests (1)
1212
# CHECK: custom-result-category :: test2.txt
1313

14-
# CHECK: My Passed Tests: 1
15-
# CHECK: My Failed Tests: 1
14+
# CHECK: My Passed: 1
15+
# CHECK: My Failed: 1

‎llvm/utils/lit/tests/googletest-discovery-failed.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@
55

66

77
# CHECK: -- Testing:
8-
# CHECK: Failing Tests (1):
8+
# CHECK: Failed Tests (1):
99
# CHECK: googletest-discovery-failed :: subdir/OneTest.py/failed_to_discover_tests_from_gtest
10-
# CHECK: Unexpected Failures: 1
10+
# CHECK: Failed: 1

‎llvm/utils/lit/tests/googletest-format.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
# CHECK: ***
1818
# CHECK: PASS: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/0.subTest
1919
# CHECK: PASS: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/1.subTest
20-
# CHECK: Failing Tests (1)
21-
# CHECK: Expected Passes : 3
22-
# CHECK: Unexpected Failures: 1
20+
# CHECK: Failed Tests (1)
21+
# CHECK: Passed: 3
22+
# CHECK: Failed: 1
2323

‎llvm/utils/lit/tests/googletest-timeout.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
# CHECK: PASS: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestA
1717
# CHECK: TIMEOUT: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB
1818
# CHECK: TIMEOUT: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestC
19-
# CHECK: Expected Passes : 1
20-
# CHECK: Individual Timeouts: 2
19+
# CHECK: Passed : 1
20+
# CHECK: Timed Out: 2
2121

2222
# Test per test timeout via a config file and on the command line.
2323
# The value set on the command line should override the config file.

‎llvm/utils/lit/tests/googletest-upstream-format.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@
1515
# CHECK: ***
1616
# CHECK: PASS: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/0.subTest
1717
# CHECK: PASS: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/1.subTest
18-
# CHECK: Failing Tests (1)
19-
# CHECK: Expected Passes : 3
20-
# CHECK: Unexpected Failures: 1
18+
# CHECK: Failed Tests (1)
19+
# CHECK: Passed: 3
20+
# CHECK: Failed: 1

‎llvm/utils/lit/tests/lit-opts.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@
2424

2525
# CHECK: Testing: 1 tests
2626
# CHECK-NOT: PASS
27-
# CHECK: Expected Passes: 1
27+
# CHECK: Passed: 1
2828

2929
# SHOW-ALL: Testing: 1 tests
3030
# SHOW-ALL: PASS: lit-opts :: test.txt (1 of 1)
3131
# SHOW-ALL: {{^}}[[VAR]]
3232
# SHOW-ALL-NOT: PASS
33-
# SHOW-ALL: Expected Passes: 1
33+
# SHOW-ALL: Passed: 1

‎llvm/utils/lit/tests/max-failures.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,15 @@
1010
#
1111

1212
# CHECK-NOT: reached maximum number of test failures
13-
# CHECK-NOT: Skipped Tests
14-
# CHECK: Unexpected Failures: 3
13+
# CHECK-NOT: Skipped
14+
# CHECK: Failed: 3
1515

1616
# CHECK: reached maximum number of test failures, skipping remaining tests
17-
# CHECK: Skipped Tests : 2
18-
# CHECK: Unexpected Failures: 1
17+
# CHECK: Skipped: 2
18+
# CHECK: Failed : 1
1919

2020
# CHECK: reached maximum number of test failures, skipping remaining tests
21-
# CHECK: Skipped Tests : 1
22-
# CHECK: Unexpected Failures: 2
21+
# CHECK: Skipped: 1
22+
# CHECK: Failed : 2
2323

2424
# CHECK: error: argument --max-failures: requires positive integer, but found '0'

‎llvm/utils/lit/tests/max-time.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@
55
# RUN: %{lit} %{inputs}/max-time --max-time=5 2>&1 | FileCheck %s
66

77
# CHECK: reached timeout, skipping remaining tests
8-
# CHECK: Skipped Tests : 1
9-
# CHECK: Expected Passes: 1
8+
# CHECK: Skipped: 1
9+
# CHECK: Passed : 1

‎llvm/utils/lit/tests/parallelism-groups.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@
1515
# CHECK: -- Testing: 2 tests, 2 workers --
1616
# CHECK-DAG: PASS: parallelism-groups :: test1.txt
1717
# CHECK-DAG: PASS: parallelism-groups :: test2.txt
18-
# CHECK: Expected Passes: 2
18+
# CHECK: Passed: 2

‎llvm/utils/lit/tests/selecting.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@
2222
# RUN: %{lit} --filter 'O[A-Z]E' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
2323
# RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
2424
# CHECK-FILTER: Testing: 2 of 5 tests
25-
# CHECK-FILTER: Excluded Tests : 3
25+
# CHECK-FILTER: Excluded: 3
2626

2727

2828
# Check that maximum counts work
2929
#
3030
# RUN: %{lit} --max-tests 3 %{inputs}/discovery | FileCheck --check-prefix=CHECK-MAX %s
3131
# CHECK-MAX: Testing: 3 of 5 tests
32-
# CHECK-MAX: Excluded Tests : 2
32+
# CHECK-MAX: Excluded: 2
3333

3434

3535
# Check that sharding partitions the testsuite in a way that distributes the
@@ -40,7 +40,7 @@
4040
# RUN: FileCheck --check-prefix=CHECK-SHARD0-OUT < %t.out %s
4141
# CHECK-SHARD0-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]
4242
# CHECK-SHARD0-OUT: Testing: 2 of 5 tests
43-
# CHECK-SHARD0-OUT: Excluded Tests : 3
43+
# CHECK-SHARD0-OUT: Excluded: 3
4444
#
4545
# RUN: %{lit} --num-shards 3 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
4646
# RUN: FileCheck --check-prefix=CHECK-SHARD1-ERR < %t.err %s

‎llvm/utils/lit/tests/shtest-env.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,6 @@
9393
# CHECK: $ "env" "A_FOO=1" "-u" "FOO" "B_BAR=2" "-u" "BAR" "C_OOF=3" "{{[^"]*}}" "print_environment.py"
9494
# CHECK-NOT: ${{.*}}print_environment.py
9595

96-
# CHECK: Expected Passes : 4
97-
# CHECK: Unexpected Failures: 12
96+
# CHECK: Passed: 4
97+
# CHECK: Failed: 12
9898
# CHECK-NOT: {{.}}

‎llvm/utils/lit/tests/shtest-format.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -69,21 +69,21 @@
6969
# CHECK-NEXT: true
7070
# CHECK-NEXT: --
7171

72-
# CHECK: Failing Tests (3)
72+
# CHECK: Failed Tests (3)
7373
# CHECK: shtest-format :: external_shell/fail.txt
7474
# CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
7575
# CHECK: shtest-format :: fail.txt
7676

77-
# CHECK: Unexpected Passing Tests (1)
77+
# CHECK: Unexpectedly Passed Tests (1)
7878
# CHECK: shtest-format :: xpass.txt
7979

8080
# CHECK: Testing Time:
81-
# CHECK: Unsupported Tests : 4
82-
# CHECK: Expected Passes : 7
83-
# CHECK: Expected Failures : 4
84-
# CHECK: Unresolved Tests : 3
85-
# CHECK: Unexpected Failures: 3
86-
# CHECK: Unexpected Passes : 1
81+
# CHECK: Unsupported : 4
82+
# CHECK: Passed : 7
83+
# CHECK: Expectedly Failed : 4
84+
# CHECK: Unresolved : 3
85+
# CHECK: Failed : 3
86+
# CHECK: Unexpectedly Passed: 1
8787

8888

8989
# XUNIT: <?xml version="1.0" encoding="UTF-8"?>

‎llvm/utils/lit/tests/shtest-inject.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
# CHECK-TEST1: THIS WAS
1212
# CHECK-TEST1: INJECTED
1313
#
14-
# CHECK-TEST1: Expected Passes: 1
14+
# CHECK-TEST1: Passed: 1
1515

1616
# RUN: %{lit} -j 1 %{inputs}/shtest-inject/test-one.txt --show-all | FileCheck --check-prefix=CHECK-TEST2 %s
1717
#
@@ -26,7 +26,7 @@
2626
# CHECK-TEST2: INJECTED
2727
# CHECK-TEST2: IN THE FILE
2828
#
29-
# CHECK-TEST2: Expected Passes: 1
29+
# CHECK-TEST2: Passed: 1
3030

3131
# RUN: %{lit} -j 1 %{inputs}/shtest-inject/test-many.txt --show-all | FileCheck --check-prefix=CHECK-TEST3 %s
3232
#
@@ -45,4 +45,4 @@
4545
# CHECK-TEST3: IF IT WORKS
4646
# CHECK-TEST3: AS EXPECTED
4747
#
48-
# CHECK-TEST3: Expected Passes: 1
48+
# CHECK-TEST3: Passed: 1

‎llvm/utils/lit/tests/shtest-not.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,6 @@
110110
# CHECK: Error: 'not --crash' cannot call 'rm'
111111
# CHECK: error: command failed with exit status: {{.*}}
112112

113-
# CHECK: Expected Passes : 1
114-
# CHECK: Unexpected Failures: 12
113+
# CHECK: Passed: 1
114+
# CHECK: Failed: 12
115115
# CHECK-NOT: {{.}}

‎llvm/utils/lit/tests/shtest-shell.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -583,4 +583,4 @@
583583
# CHECK: ***
584584

585585
# CHECK: PASS: shtest-shell :: valid-shell.txt
586-
# CHECK: Failing Tests (35)
586+
# CHECK: Failed Tests (35)

‎llvm/utils/lit/tests/shtest-timeout.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@
5050

5151
# CHECK-OUT-COMMON: PASS: per_test_timeout :: short.py
5252

53-
# CHECK-OUT-COMMON: Expected Passes{{ *}}: 1
54-
# CHECK-OUT-COMMON: Individual Timeouts{{ *}}: 1
53+
# CHECK-OUT-COMMON: Passed : 1
54+
# CHECK-OUT-COMMON: Timed Out: 1
5555

5656
# Test per test timeout via a config file and on the command line.
5757
# The value set on the command line should override the config file.
@@ -71,5 +71,5 @@
7171

7272
# CHECK-CMDLINE-OVERRIDE-OUT: PASS: per_test_timeout :: short.py
7373

74-
# CHECK-CMDLINE-OVERRIDE-OUT: Expected Passes{{ *}}: 1
75-
# CHECK-CMDLINE-OVERRIDE-OUT: Individual Timeouts{{ *}}: 1
74+
# CHECK-CMDLINE-OVERRIDE-OUT: Passed : 1
75+
# CHECK-CMDLINE-OVERRIDE-OUT: Timed Out: 1

‎mlir/test/Examples/standalone/test.toy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
# RUN: %cmake %mlir_src_root/examples/standalone -DCMAKE_CXX_COMPILER=%host_cxx -DCMAKE_C_COMPILER=%host_cc -DMLIR_DIR=%llvm_lib_dir/cmake/mlir ; %cmake --build . --target check-standalone | tee %t | FileCheck %s
22

3-
# CHECK: Expected Passes: 3
3+
# CHECK: Passed: 3
44
# UNSUPPORTED: windows, android

0 commit comments

Comments
 (0)