Skip to content

Commit a534127

Browse files
jvdpnrdxpjonringer
authored
ETCM-680 retesteth in Nix and on CI (#965)
Co-authored-by: Timothy DeHerrera <[email protected]> Co-authored-by: Jonathan Ringer <[email protected]>
1 parent 8f48ec4 commit a534127

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+491
-1380
lines changed

‎.buildkite/pipeline.nix

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,13 @@ in
136136
command = ''
137137
nix-shell --run './test-ets.sh'
138138
'';
139+
softFail = true;
140+
retry.automatic = false;
141+
artifactPaths = [
142+
"mantis-log.txt"
143+
"retesteth-GeneralStateTests-log.txt"
144+
"retesteth-BlockchainTests-log.txt"
145+
];
139146
};
140147

141148
coverageReport = commonAttrs // {

‎.buildkite/shell.nix

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
{ sources, pkgs }:
2-
1+
{ sources, pkgs }:
32
let
4-
53
# TODO, share this code with mantis build in this project
64
# sbt-protoc puts the scala plugin in /tmp/protobridge<some-random-number>.
75
# it is in fact a shell script with a standard `#!/usr/bin/env sh` shebang
@@ -18,11 +16,10 @@ let
1816
exec ${pkgs.protobuf}/bin/protoc "$@"
1917
'';
2018
in
19+
with pkgs;
2120

22-
with pkgs;
23-
24-
mkShell {
25-
nativeBuildInputs = [ sbt solc jdk8 protoc-wrapper ];
26-
# SBT = "sbt -v -mem 2048 -J-Xmx4g -Dsbt.ivy.home=/cache/ivy2 -Dsbt.boot.directory=/cache/sbt -Dmaven.repo.local=/cache/maven -Dnix=true";
27-
SBT = "sbt -v -mem 2048 -J-Xmx4g -Dnix=true";
28-
}
21+
mkShell {
22+
nativeBuildInputs = [ sbt solc lllc jdk8 protoc-wrapper retesteth netcat-gnu ];
23+
# SBT = "sbt -v -mem 2048 -J-Xmx4g -Dsbt.ivy.home=/cache/ivy2 -Dsbt.boot.directory=/cache/sbt -Dmaven.repo.local=/cache/maven -Dnix=true";
24+
SBT = "sbt -v -mem 2048 -J-Xmx4g -Dnix=true";
25+
}

‎.gitmodules

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
[submodule "src/ets/resources/ets"]
2-
path = src/ets/resources/ets
1+
[submodule "ets/tests"]
2+
path = ets/tests
33
url = https://github.com/ethereum/tests
4+
45
[submodule "src/main/protobuf/extvm"]
56
path = src/main/protobuf/extvm
67
url = https://github.com/input-output-hk/mantis-extvm-pb

‎README.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,13 @@ In the root of the project:
119119
nix-build
120120
```
121121

122+
##### On a Mac
123+
124+
This project uses Nix for CI, deployment and, optionally, local development.
125+
Some of the dependencies are not available for Darwin (macOS) however. To work
126+
with Nix on a Mac you can instead use Docker via the `nix-in-docker/run` script,
127+
which will start a `nix-shell` with the same environment as CI.
128+
122129
##### Update sbt+nix dependencies
123130

124131
When updating project dependencies, the nix fixed-output-derivation

‎build.sbt

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,6 @@ lazy val node = {
135135

136136
val Evm = config("evm") extend Test
137137

138-
val Ets = config("ets") extend Test
139-
140138
val Rpc = config("rpcTest") extend Test
141139

142140
val malletDeps = Seq(
@@ -182,7 +180,7 @@ lazy val node = {
182180

183181
val node = project
184182
.in(file("."))
185-
.configs(Integration, Benchmark, Evm, Ets, Rpc)
183+
.configs(Integration, Benchmark, Evm, Rpc)
186184
.enablePlugins(BuildInfoPlugin)
187185
.dependsOn(bytes, crypto, rlp)
188186
.settings(
@@ -217,7 +215,6 @@ lazy val node = {
217215
)
218216
.settings(inConfig(Benchmark)(Defaults.testSettings :+ (Test / parallelExecution := false)): _*)
219217
.settings(inConfig(Evm)(Defaults.testSettings :+ (Test / parallelExecution := false)): _*)
220-
.settings(inConfig(Ets)(Defaults.testSettings :+ (Test / parallelExecution := false)): _*)
221218
.settings(inConfig(Rpc)(Defaults.testSettings :+ (Test / parallelExecution := false)): _*)
222219
.settings(
223220
// protobuf compilation
@@ -266,7 +263,6 @@ addCommandAlias(
266263
|;test:compile
267264
|;evm:compile
268265
|;it:compile
269-
|;ets:compile
270266
|;rpcTest:compile
271267
|;benchmark:compile
272268
|""".stripMargin

‎ci-like/Dockerfile

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎ets/README.md

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Ethereum Test Suite
2+
3+
This folder contains a git submodule pointing at the Ethereum Consensus Tests,
4+
also known as the Ethereum Test Suite (ETS), config files for retesteth (the
5+
tool for running these tests) and a wrapper script to set the its command line
6+
options. Oh, and this readme file is in there too, of course.
7+
8+
* ETS: https://github.com/ethereum/tests
9+
* retesteth: https://github.com/ethereum/retesteth
10+
11+
## Continous integration
12+
13+
The tests are run on CI. For more details look at `.buildkite/pipeline.nix` and
14+
`test-ets.sh`. Output is stored as artifacts and a summary is added as
15+
annotation.
16+
17+
Two test suites are run; GeneralStateTests and BlockchainTests. These seem to
18+
be the only ones maintained and recommended at the moment.
19+
20+
## Running ETS in a Nix environment
21+
22+
Start Mantis in test mode:
23+
24+
sbt -Dconfig.file=./src/main/resources/conf/testmode.conf -Dlogging.logs-level=WARN run
25+
26+
NB. raising the log level is a good idea as there will be a lot of output,
27+
depending on how many tests you run.
28+
29+
Once the RPC API is up, run retesteth:
30+
31+
ets/retesteth -t GeneralStateTests
32+
33+
You can also run parts of the suite; refer to `ets/retesteth --help` for details.
34+
35+
## Running retesteth in Docker (eg. macOS)
36+
37+
You should run Mantis outside Nix as that is probably more convenient for your
38+
tooling (eg. attaching a debugger.)
39+
40+
sbt -Dconfig.file=./src/main/resources/conf/testmode.conf -Dlogging.logs-level=WARN run
41+
42+
Retesteth will need to be able to connect to Mantis, running on the host
43+
system. First, find the IP it should use:
44+
45+
nix-in-docker/run --command "getent hosts host.docker.internal"
46+
47+
Edit `ets/config/mantis` and replace `0.0.0.0` with this IP.
48+
49+
Finally, run retesteth in Nix in Docker:
50+
51+
nix-in-docker/run --command "ets/retesteth -t GeneralStateTests"

‎ets/config/mantis/config

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"name" : "IOHK Mantis on TCP",
3+
"socketType" : "tcp",
4+
"socketAddress" : [
5+
"0.0.0.0:8546"
6+
],
7+
"initializeTime" : "5",
8+
"forks" : [
9+
"Frontier",
10+
"Homestead",
11+
"EIP150",
12+
"EIP158",
13+
"Byzantium",
14+
"Constantinople",
15+
"ConstantinopleFix",
16+
"Istanbul"
17+
],
18+
"additionalForks" : [
19+
"FrontierToHomesteadAt5",
20+
"HomesteadToEIP150At5",
21+
"EIP158ToByzantiumAt5",
22+
"HomesteadToDaoAt5",
23+
"ByzantiumToConstantinopleFixAt5"
24+
],
25+
"exceptions" : {}
26+
}

‎ets/config/mantis/genesis/Berlin.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00",
5+
"EIP158ForkBlock" : "0x00",
6+
"byzantiumForkBlock" : "0x00",
7+
"constantinopleForkBlock" : "0x00",
8+
"constantinopleFixForkBlock" : "0x00",
9+
"istanbulForkBlock" : "0x00",
10+
"berlinForkBlock" : "0x00",
11+
"chainID" : "0x01"
12+
},
13+
"accounts" : {
14+
}
15+
}

‎ets/config/mantis/genesis/Byzantium.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00",
5+
"EIP158ForkBlock" : "0x00",
6+
"byzantiumForkBlock" : "0x00"
7+
},
8+
"accounts" : {
9+
}
10+
}

‎ets/config/mantis/genesis/ByzantiumToConstantinopleFixAt5.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00",
5+
"EIP158ForkBlock" : "0x00",
6+
"byzantiumForkBlock" : "0x00",
7+
"constantinopleForkBlock" : "0x05",
8+
"constantinopleFixForkBlock" : "0x05"
9+
},
10+
"accounts" : {
11+
}
12+
}

‎ets/config/mantis/genesis/Constantinople.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00",
5+
"EIP158ForkBlock" : "0x00",
6+
"byzantiumForkBlock" : "0x00",
7+
"constantinopleForkBlock" : "0x00"
8+
},
9+
"accounts" : {
10+
}
11+
}

‎ets/config/mantis/genesis/ConstantinopleFix.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00",
5+
"EIP158ForkBlock" : "0x00",
6+
"byzantiumForkBlock" : "0x00",
7+
"constantinopleForkBlock" : "0x00",
8+
"constantinopleFixForkBlock" : "0x00"
9+
},
10+
"accounts" : {
11+
}
12+
}

‎ets/config/mantis/genesis/EIP150.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00"
5+
},
6+
"accounts" : {
7+
}
8+
}

‎ets/config/mantis/genesis/EIP158.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00",
5+
"EIP158ForkBlock" : "0x00"
6+
},
7+
"accounts" : {
8+
}
9+
}

‎ets/config/mantis/genesis/EIP158ToByzantiumAt5.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00",
5+
"EIP158ForkBlock" : "0x00",
6+
"byzantiumForkBlock" : "0x05"
7+
},
8+
"accounts" : {
9+
}
10+
}

‎ets/config/mantis/genesis/Frontier.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"params" : {
3+
},
4+
"accounts" : {
5+
}
6+
}

‎ets/config/mantis/genesis/FrontierToHomesteadAt5.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x05"
4+
},
5+
"accounts" : {
6+
}
7+
}

‎ets/config/mantis/genesis/Homestead.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00"
4+
},
5+
"accounts" : {
6+
}
7+
}

‎ets/config/mantis/genesis/HomesteadToDaoAt5.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"daoHardforkBlock" : "0x05"
5+
},
6+
"accounts" : {
7+
}
8+
}

‎ets/config/mantis/genesis/HomesteadToEIP150At5.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x05"
5+
},
6+
"accounts" : {
7+
}
8+
}

‎ets/config/mantis/genesis/Istanbul.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"params" : {
3+
"homesteadForkBlock" : "0x00",
4+
"EIP150ForkBlock" : "0x00",
5+
"EIP158ForkBlock" : "0x00",
6+
"byzantiumForkBlock" : "0x00",
7+
"constantinopleForkBlock" : "0x00",
8+
"constantinopleFixForkBlock" : "0x00",
9+
"istanbulForkBlock" : "0x00",
10+
"chainID" : "0x01"
11+
},
12+
"accounts" : {
13+
}
14+
}

‎ets/config/mantis/genesis/correctMiningReward.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"//comment" : "State Tests does not calculate mining reward in post conditions, so when filling a blockchain test out of it, the mining reward must be set",
3+
"Frontier": "5000000000000000000",
4+
"Homestead": "5000000000000000000",
5+
"EIP150": "5000000000000000000",
6+
"EIP158": "5000000000000000000",
7+
"Byzantium": "3000000000000000000",
8+
"Constantinople": "2000000000000000000",
9+
"ConstantinopleFix": "2000000000000000000",
10+
"Istanbul": "2000000000000000000",
11+
"Berlin" : "2000000000000000000"
12+
}

‎ets/config/version

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
-

‎ets/retesteth

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env sh
2+
3+
dir="$(cd "$(dirname "$0")" && pwd)"
4+
5+
retesteth $@ -- --testpath $dir/tests --datadir $dir/config --clients mantis

‎ets/tests

Submodule tests added at 644967e

‎flake.nix

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,7 @@
4949
}) // (collectHydraSets [
5050
(mkHydraSet [ "mantis" ] [ "x86_64-linux" ])
5151
(mkHydraSet [ "mantis-entrypoint" ] [ "x86_64-linux" ])
52+
(mkHydraSet [ "retesteth" ] [ "x86_64-linux" ])
53+
(mkHydraSet [ "lllc" ] [ "x86_64-linux" ])
5254
]);
5355
}

‎ci-like/nix.conf renamed to ‎nix-in-docker/nix.conf

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
sandbox = false
1+
sandbox = relaxed
22
experimental-features = nix-command flakes ca-references
3-
substituters = https://hydra.iohk.io https://cache.nixos.org https://mantis-ops.cachix.org
4-
trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ=
3+
substituters = https://hydra.iohk.io https://cache.nixos.org https://mantis-ops.cachix.org https://hydra.mantis.ist
4+
trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ= hydra.mantis.ist-1:4LTe7Q+5pm8+HawKxvmn2Hx0E3NbkYjtf1oWv+eAmTo=

0 commit comments

Comments
 (0)