Skip to content

Commit 9b848e2

Browse files
anvacarurv-auditor
anvacaru
and
rv-auditor
authored
Tweaks to the conformance harness (#2505)
* tweaks to the conformance harness * Set Version: 1.0.621 * apply review suggestions * bchain test ids relative to BlockchainTests/GeneralStateTests/ --------- Co-authored-by: devops <[email protected]>
1 parent 9443a79 commit 9b848e2

File tree

5 files changed

+21
-110
lines changed

5 files changed

+21
-110
lines changed

‎kevm-pyk/pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
66
name = "kevm-pyk"
7-
version = "1.0.620"
7+
version = "1.0.621"
88
description = ""
99
authors = [
1010
"Runtime Verification, Inc. <[email protected]>",

‎kevm-pyk/src/kevm_pyk/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@
55
if TYPE_CHECKING:
66
from typing import Final
77

8-
VERSION: Final = '1.0.620'
8+
VERSION: Final = '1.0.621'

‎kevm-pyk/src/tests/integration/test_conformance.py

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,16 @@
3434

3535

3636
def _test(gst_file: Path, schedule: str, mode: str, chainid: int, usegas: bool) -> None:
37+
skipped_gst_tests = SKIPPED_TESTS.get(gst_file, [])
38+
if '*' in skipped_gst_tests:
39+
pytest.skip()
40+
3741
with gst_file.open() as f:
3842
gst_data = json.load(f)
3943

4044
for test_name, test in gst_data.items():
4145
_LOGGER.info(f'Running test: {gst_file} - {test_name}')
42-
if test_name in SKIPPED_TESTS.get(gst_file, []):
46+
if test_name in skipped_gst_tests:
4347
continue
4448
res = interpret({test_name: test}, schedule, mode, chainid, usegas, check=False)
4549
_assert_exit_code_zero(res)
@@ -79,9 +83,8 @@ def read_csv_file(csv_file: Path) -> tuple[tuple[Path, str], ...]:
7983
SKIPPED_TESTS: Final = _skipped_tests()
8084

8185
VM_TEST_DIR: Final = TEST_DIR / 'BlockchainTests/GeneralStateTests/VMTests'
82-
ALL_VM_TESTS: Final = tuple(VM_TEST_DIR.glob('*/*.json'))
83-
VM_TESTS: Final = tuple(test_file for test_file in ALL_VM_TESTS if test_file not in SKIPPED_TESTS)
84-
REST_VM_TESTS: Final = tuple(test_file for test_file in ALL_VM_TESTS if test_file in SKIPPED_TESTS)
86+
VM_TESTS: Final = tuple(VM_TEST_DIR.glob('*/*.json'))
87+
SKIPPED_VM_TESTS: Final = tuple(test_file for test_file in VM_TESTS if test_file in SKIPPED_TESTS)
8588

8689

8790
@pytest.mark.parametrize(
@@ -96,27 +99,23 @@ def test_vm(test_file: Path) -> None:
9699
@pytest.mark.skip(reason='failing / slow VM tests')
97100
@pytest.mark.parametrize(
98101
'test_file',
99-
REST_VM_TESTS,
100-
ids=[str(test_file.relative_to(VM_TEST_DIR)) for test_file in REST_VM_TESTS],
102+
SKIPPED_VM_TESTS,
103+
ids=[str(test_file.relative_to(VM_TEST_DIR)) for test_file in SKIPPED_VM_TESTS],
101104
)
102105
def test_rest_vm(test_file: Path) -> None:
103106
_test(test_file, 'DEFAULT', 'VMTESTS', 1, True)
104107

105108

106-
BCHAIN_TEST_DIR: Final = TEST_DIR / 'BlockchainTests/GeneralStateTests'
107-
ALL_BCHAIN_TESTS: Final = tuple(BCHAIN_TEST_DIR.glob('**/*.json'))
108-
BCHAIN_TESTS: Final = tuple(
109-
test_file for test_file in ALL_BCHAIN_TESTS if test_file not in (list(SKIPPED_TESTS) + list(ALL_VM_TESTS))
110-
)
111-
REST_BCHAIN_TESTS: Final = tuple(
112-
test_file for test_file in ALL_BCHAIN_TESTS if test_file in SKIPPED_TESTS and test_file not in ALL_VM_TESTS
113-
)
109+
ALL_TEST_DIR: Final = TEST_DIR / 'BlockchainTests/GeneralStateTests'
110+
ALL_TESTS: Final = tuple(ALL_TEST_DIR.glob('**/*.json'))
111+
BCHAIN_TESTS: Final = tuple(test_file for test_file in ALL_TESTS if test_file not in set(VM_TESTS))
112+
SKIPPED_BCHAIN_TESTS: Final = tuple(test_file for test_file in BCHAIN_TESTS if test_file in SKIPPED_TESTS)
114113

115114

116115
@pytest.mark.parametrize(
117116
'test_file',
118117
BCHAIN_TESTS,
119-
ids=[str(test_file.relative_to(TEST_DIR)) for test_file in BCHAIN_TESTS],
118+
ids=[str(test_file.relative_to(ALL_TEST_DIR)) for test_file in BCHAIN_TESTS],
120119
)
121120
def test_bchain(test_file: Path) -> None:
122121
_test(test_file, 'SHANGHAI', 'NORMAL', 1, True)
@@ -125,8 +124,8 @@ def test_bchain(test_file: Path) -> None:
125124
@pytest.mark.skip(reason='failing / slow blockchain tests')
126125
@pytest.mark.parametrize(
127126
'test_file',
128-
REST_BCHAIN_TESTS,
129-
ids=[str(test_file.relative_to(TEST_DIR)) for test_file in REST_BCHAIN_TESTS],
127+
SKIPPED_BCHAIN_TESTS,
128+
ids=[str(test_file.relative_to(ALL_TEST_DIR)) for test_file in SKIPPED_BCHAIN_TESTS],
130129
)
131130
def test_rest_bchain(test_file: Path) -> None:
132131
_test(test_file, 'SHANGHAI', 'NORMAL', 1, True)

‎package/version

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.0.620
1+
1.0.621

‎tests/slow.llvm

Lines changed: 2 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -1,90 +1,2 @@
1-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_Berlin
2-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_Istanbul
3-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_London
4-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_Merge
5-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d0g0v0_Shanghai
6-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_Berlin
7-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_Istanbul
8-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_London
9-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_Merge
10-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d1g0v0_Shanghai
11-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_Berlin
12-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_Istanbul
13-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_London
14-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_Merge
15-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,loopMul_d2g0v0_Shanghai
16-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_Berlin
17-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_Istanbul
18-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_London
19-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_Merge
20-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d0g0v0_Shanghai
21-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_Berlin
22-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_Istanbul
23-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_London
24-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_Merge
25-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d10g0v0_Shanghai
26-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_Berlin
27-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_Istanbul
28-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_London
29-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_Merge
30-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d11g0v0_Shanghai
31-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_Berlin
32-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_Istanbul
33-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_London
34-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_Merge
35-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d12g0v0_Shanghai
36-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_Berlin
37-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_Istanbul
38-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_London
39-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_Merge
40-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d13g0v0_Shanghai
41-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_Berlin
42-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_Istanbul
43-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_London
44-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_Merge
45-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d14g0v0_Shanghai
46-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_Berlin
47-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_Istanbul
48-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_London
49-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_Merge
50-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d1g0v0_Shanghai
51-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_Berlin
52-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_Istanbul
53-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_London
54-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_Merge
55-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d2g0v0_Shanghai
56-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_Berlin
57-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_Istanbul
58-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_London
59-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_Merge
60-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d3g0v0_Shanghai
61-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_Berlin
62-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_Istanbul
63-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_London
64-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_Merge
65-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d4g0v0_Shanghai
66-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_Berlin
67-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_Istanbul
68-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_London
69-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_Merge
70-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d5g0v0_Shanghai
71-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_Berlin
72-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_Istanbul
73-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_London
74-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_Merge
75-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d6g0v0_Shanghai
76-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_Berlin
77-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_Istanbul
78-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_London
79-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_Merge
80-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d7g0v0_Shanghai
81-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_Berlin
82-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_Istanbul
83-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_London
84-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_Merge
85-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d8g0v0_Shanghai
86-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_Berlin
87-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_Istanbul
88-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_London
89-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_Merge
90-
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,loopExp_d9g0v0_Shanghai
1+
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopMul.json,*
2+
BlockchainTests/GeneralStateTests/VMTests/vmPerformance/loopExp.json,*

0 commit comments

Comments
 (0)