Skip to content

Commit 4d2bd58

Browse files
committed
Merge branch 'ETCM-311-fast_sync_improvements' of github.com:input-output-hk/mantis into ETCM-311-fast_sync_improvements
2 parents e2ae8bf + e59e0a0 commit 4d2bd58

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+799
-268
lines changed

.buildkite/pipeline.nix

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
{ cfg, pkgs, ... }:
22

33
with cfg.steps.commands;
4-
54
let
65
commonAttrs = {
76
retry.automatic = true;
87
agents.queue = "project42";
98
};
109
in
11-
1210
{
1311
steps.commands = {
1412
nixExpr = commonAttrs // {
@@ -97,6 +95,19 @@ in
9795
];
9896
};
9997

98+
annotate-test-reports = commonAttrs // {
99+
dependsOn = [ test-unit ];
100+
label = "annotate test reports";
101+
command = "junit-annotate";
102+
allowDependencyFailure = true;
103+
plugins = [{
104+
"junit-annotate#1.9.0" = {
105+
artifacts = "target/test-reports/*.xml";
106+
report-slowest = 50;
107+
};
108+
}];
109+
};
110+
100111
test-evm = commonAttrs // {
101112
dependsOn = [ compile ];
102113
label = "EVM tests";

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,6 @@ metals.sbt
2424
# Nix
2525
result
2626
.nix/
27+
28+
# sonarScan
29+
.scannerwork/

build.sbt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,3 +255,6 @@ addCommandAlias(
255255
|;it:test
256256
|""".stripMargin
257257
)
258+
259+
scapegoatVersion in ThisBuild := "1.4.7"
260+
scapegoatReports := Seq("xml")

ci-like/Dockerfile

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM nixos/nix
2+
3+
RUN apk --update add git less openssh && \
4+
rm -rf /var/lib/apt/lists/* && \
5+
rm /var/cache/apk/*
6+
7+
RUN nix-channel --add https://nixos.org/channels/nixpkgs-unstable nixpkgs
8+
RUN nix-channel --update
9+
10+
RUN nix-build -A pythonFull '<nixpkgs>'
11+

ci-like/nix.conf

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
sandbox = false
2+
experimental-features = nix-command flakes ca-references
3+
substituters = https://hydra.iohk.io https://cache.nixos.org https://mantis-ops.cachix.org
4+
trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ=

nix/pkgs/mantis.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ in sbt.mkDerivation {
5050

5151
# This sha represents the change dependencies of mantis.
5252
# Update this sha whenever you change the dependencies
53-
depsSha256 = "07iixw8va4zwpiln2zy2gr245z1ir4jd6pqgmkzhwnhw3mf5j28k";
53+
depsSha256 = "14hx1gxa7505b8jy1vq5gc5p51fn80sj0pafx26awsrl6q67qyld";
5454

5555
# this is the command used to to create the fixed-output-derivation
5656
depsWarmupCommand = "sbt compile --debug -Dnix=true";

project/Dependencies.scala

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ object Dependencies {
4848

4949
val rocksDb = Seq(
5050
// use "5.18.3" for older macOS
51-
"org.rocksdb" % "rocksdbjni" % "6.11.4"
51+
"org.rocksdb" % "rocksdbjni" % "6.15.2"
5252
)
5353

5454
val enumeratum: Seq[ModuleID] = Seq(
@@ -59,7 +59,7 @@ object Dependencies {
5959

6060
val testing: Seq[ModuleID] = Seq(
6161
"org.scalatest" %% "scalatest" % "3.2.2" % "it,test",
62-
"org.scalamock" %% "scalamock" % "5.0.0" % "test",
62+
"org.scalamock" %% "scalamock" % "5.0.0" % "it,test",
6363
"org.scalatestplus" %% "scalacheck-1-15" % "3.2.3.0" % "test",
6464
"org.scalacheck" %% "scalacheck" % "1.15.1" % "it,test",
6565
"com.softwaremill.diffx" %% "diffx-core" % "0.3.30" % "test",
@@ -114,7 +114,11 @@ object Dependencies {
114114
"org.scala-sbt.ipcsocket" % "ipcsocket" % "1.1.0",
115115
"org.xerial.snappy" % "snappy-java" % "1.1.7.7",
116116
"org.web3j" % "core" % "5.0.0" % Test,
117-
"io.vavr" % "vavr" % "1.0.0-alpha-3"
117+
"io.vavr" % "vavr" % "1.0.0-alpha-3",
118+
"org.jupnp" % "org.jupnp" % "2.5.2",
119+
"org.jupnp" % "org.jupnp.support" % "2.5.2",
120+
"org.jupnp" % "org.jupnp.tool" % "2.5.2",
121+
"javax.servlet" % "javax.servlet-api" % "4.0.1"
118122
)
119123

120124
val guava: Seq[ModuleID] = {

project/plugins.sbt

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
logLevel := sbt.Level.Warn
2-
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1")
3-
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2")
4-
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
5-
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.5")
2+
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0")
3+
addSbtPlugin("com.github.mwz" % "sbt-sonar" % "2.2.0")
4+
addSbtPlugin("com.lightbend.sbt" % "sbt-javaagent" % "0.1.6")
5+
addSbtPlugin("com.sksamuel.scapegoat" % "sbt-scapegoat" % "1.1.0")
66
addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.0")
77
addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.5.1")
8-
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0")
98
addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0")
9+
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.5")
1010
addSbtPlugin("io.kamon" % "sbt-kanela-runner" % "2.0.5")
11-
addSbtPlugin("com.lightbend.sbt" % "sbt-javaagent" % "0.1.6")
11+
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2")
12+
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
13+
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1")

scalastyle-config.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<scalastyle>
22
<name>Scalastyle standard configuration</name>
33
<check level="error" class="org.scalastyle.file.FileTabChecker" enabled="true"></check>
4-
<check level="error" class="org.scalastyle.file.FileLengthChecker" enabled="true">
4+
<check level="error" class="org.scalastyle.file.FileLengthChecker" enabled="false">
55
<parameters>
66
<parameter name="maxFileLength"><![CDATA[800]]></parameter>
77
</parameters>
Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
package io.iohk.ethereum.ledger
2+
3+
import akka.testkit.TestProbe
4+
import akka.util.ByteString
5+
import cats.data.NonEmptyList
6+
import io.iohk.ethereum.blockchain.sync.regular.BlockImporter.NewCheckpoint
7+
import io.iohk.ethereum.blockchain.sync.regular.{BlockFetcher, BlockImporter}
8+
import io.iohk.ethereum.checkpointing.CheckpointingTestHelpers
9+
import io.iohk.ethereum.consensus.blocks.CheckpointBlockGenerator
10+
import io.iohk.ethereum.domain._
11+
import io.iohk.ethereum.mpt.MerklePatriciaTrie
12+
import io.iohk.ethereum.utils.Config.SyncConfig
13+
import io.iohk.ethereum.utils.Config
14+
import io.iohk.ethereum.{Fixtures, ObjectGenerators, crypto}
15+
import io.iohk.ethereum.ledger.Ledger.BlockResult
16+
import monix.execution.Scheduler
17+
import org.scalamock.scalatest.MockFactory
18+
import org.scalatest.BeforeAndAfterAll
19+
import org.scalatest.flatspec.AsyncFlatSpecLike
20+
import org.scalatest.matchers.should.Matchers
21+
22+
import scala.concurrent.duration._
23+
24+
class BlockImporterItSpec extends MockFactory with TestSetupWithVmAndValidators with AsyncFlatSpecLike with Matchers with BeforeAndAfterAll {
25+
26+
implicit val testScheduler = Scheduler.fixedPool("test", 32)
27+
28+
override def afterAll(): Unit = {
29+
testScheduler.shutdown()
30+
testScheduler.awaitTermination(60.second)
31+
}
32+
33+
val blockQueue = BlockQueue(blockchain, SyncConfig(Config.config))
34+
35+
val genesis = Block(
36+
Fixtures.Blocks.Genesis.header.copy(stateRoot = ByteString(MerklePatriciaTrie.EmptyRootHash)),
37+
Fixtures.Blocks.Genesis.body
38+
)
39+
val genesisWeight = ChainWeight.zero.increase(genesis.header)
40+
41+
blockchain.save(genesis, Seq(), genesisWeight, saveAsBestBlock = true)
42+
43+
lazy val checkpointBlockGenerator: CheckpointBlockGenerator = new CheckpointBlockGenerator
44+
45+
val fetcherProbe = TestProbe()
46+
val ommersPoolProbe = TestProbe()
47+
val broadcasterProbe = TestProbe()
48+
val pendingTransactionsManagerProbe = TestProbe()
49+
val supervisor = TestProbe()
50+
51+
val emptyWorld: InMemoryWorldStateProxy =
52+
blockchain.getWorldStateProxy(
53+
-1,
54+
UInt256.Zero,
55+
ByteString(MerklePatriciaTrie.EmptyRootHash),
56+
noEmptyAccounts = false,
57+
ethCompatibleStorage = true
58+
)
59+
60+
override lazy val ledger = new TestLedgerImpl(successValidators) {
61+
override private[ledger] lazy val blockExecution = new BlockExecution(blockchain, blockchainConfig, consensus.blockPreparator, blockValidation) {
62+
override def executeAndValidateBlock(block: Block, alreadyValidated: Boolean = false): Either[BlockExecutionError, Seq[Receipt]] =
63+
Right(BlockResult(emptyWorld).receipts)
64+
}
65+
}
66+
67+
val blockImporter = system.actorOf(
68+
BlockImporter.props(
69+
fetcherProbe.ref,
70+
ledger,
71+
blockchain,
72+
syncConfig,
73+
ommersPoolProbe.ref,
74+
broadcasterProbe.ref,
75+
pendingTransactionsManagerProbe.ref,
76+
checkpointBlockGenerator,
77+
supervisor.ref
78+
))
79+
80+
val genesisBlock = blockchain.genesisBlock
81+
val block1: Block = getBlock(genesisBlock.number + 1, parent = genesisBlock.header.hash)
82+
// new chain is shorter but has a higher weight
83+
val newBlock2: Block = getBlock(genesisBlock.number + 2, difficulty = 108, parent = block1.header.hash)
84+
val newBlock3: Block = getBlock(genesisBlock.number + 3, difficulty = 300, parent = newBlock2.header.hash)
85+
val oldBlock2: Block = getBlock(genesisBlock.number + 2, difficulty = 102, parent = block1.header.hash)
86+
val oldBlock3: Block = getBlock(genesisBlock.number + 3, difficulty = 103, parent = oldBlock2.header.hash)
87+
val oldBlock4: Block = getBlock(genesisBlock.number + 4, difficulty = 104, parent = oldBlock3.header.hash)
88+
89+
val weight1 = ChainWeight.totalDifficultyOnly(block1.header.difficulty)
90+
val newWeight2 = weight1.increase(newBlock2.header)
91+
val newWeight3 = newWeight2.increase(newBlock3.header)
92+
val oldWeight2 = weight1.increase(oldBlock2.header)
93+
val oldWeight3 = oldWeight2.increase(oldBlock3.header)
94+
val oldWeight4 = oldWeight3.increase(oldBlock4.header)
95+
96+
//saving initial main chain
97+
blockchain.save(block1, Nil, weight1, saveAsBestBlock = true)
98+
blockchain.save(oldBlock2, Nil, oldWeight2, saveAsBestBlock = true)
99+
blockchain.save(oldBlock3, Nil, oldWeight3, saveAsBestBlock = true)
100+
blockchain.save(oldBlock4, Nil, oldWeight4, saveAsBestBlock = true)
101+
102+
val oldBranch = List(oldBlock2, oldBlock3, oldBlock4)
103+
val newBranch = List(newBlock2, newBlock3)
104+
105+
blockImporter ! BlockImporter.Start
106+
107+
"BlockImporter" should "not discard blocks of the main chain if the reorganisation failed" in {
108+
109+
//ledger with not mocked blockExecution
110+
val ledger = new TestLedgerImpl(successValidators)
111+
val blockImporter = system.actorOf(
112+
BlockImporter.props(
113+
fetcherProbe.ref,
114+
ledger,
115+
blockchain,
116+
syncConfig,
117+
ommersPoolProbe.ref,
118+
broadcasterProbe.ref,
119+
pendingTransactionsManagerProbe.ref,
120+
checkpointBlockGenerator,
121+
supervisor.ref
122+
))
123+
124+
blockImporter ! BlockImporter.Start
125+
blockImporter ! BlockFetcher.PickedBlocks(NonEmptyList.fromListUnsafe(newBranch))
126+
127+
Thread.sleep(1000)
128+
//because the blocks are not valid, we shouldn't reorganise, but at least stay with a current chain, and the best block of the current chain is oldBlock4
129+
blockchain.getBestBlock().get shouldEqual oldBlock4
130+
}
131+
132+
it should "return a correct new best block after reorganising longer chain to a shorter one if its weight is bigger" in {
133+
134+
//returning discarded initial chain
135+
blockchain.save(oldBlock2, Nil, oldWeight2, saveAsBestBlock = true)
136+
blockchain.save(oldBlock3, Nil, oldWeight3, saveAsBestBlock = true)
137+
blockchain.save(oldBlock4, Nil, oldWeight4, saveAsBestBlock = true)
138+
139+
blockImporter ! BlockFetcher.PickedBlocks(NonEmptyList.fromListUnsafe(newBranch))
140+
141+
Thread.sleep(200)
142+
blockchain.getBestBlock().get shouldEqual newBlock3
143+
}
144+
145+
146+
it should "switch to a branch with a checkpoint" in {
147+
148+
val checkpoint = ObjectGenerators.fakeCheckpointGen(3, 3).sample.get
149+
val oldBlock5WithCheckpoint: Block = checkpointBlockGenerator.generate(oldBlock4, checkpoint)
150+
blockchain.save(oldBlock5WithCheckpoint, Nil, oldWeight4, saveAsBestBlock = true)
151+
152+
val newBranch = List(newBlock2, newBlock3)
153+
154+
blockImporter ! BlockFetcher.PickedBlocks(NonEmptyList.fromListUnsafe(newBranch))
155+
156+
Thread.sleep(200)
157+
blockchain.getBestBlock().get shouldEqual oldBlock5WithCheckpoint
158+
blockchain.getLatestCheckpointBlockNumber() shouldEqual oldBlock5WithCheckpoint.header.number
159+
}
160+
161+
it should "switch to a branch with a newer checkpoint" in {
162+
163+
val checkpoint = ObjectGenerators.fakeCheckpointGen(3, 3).sample.get
164+
val newBlock4WithCheckpoint: Block = checkpointBlockGenerator.generate(newBlock3, checkpoint)
165+
blockchain.save(newBlock4WithCheckpoint, Nil, newWeight3, saveAsBestBlock = true)
166+
167+
val newBranch = List(newBlock4WithCheckpoint)
168+
169+
blockImporter ! BlockFetcher.PickedBlocks(NonEmptyList.fromListUnsafe(newBranch))
170+
171+
Thread.sleep(200)
172+
blockchain.getBestBlock().get shouldEqual newBlock4WithCheckpoint
173+
blockchain.getLatestCheckpointBlockNumber() shouldEqual newBlock4WithCheckpoint.header.number
174+
}
175+
176+
it should "return a correct checkpointed block after receiving a request for generating a new checkpoint" in {
177+
178+
val parent = blockchain.getBestBlock().get
179+
val newBlock5: Block = getBlock(genesisBlock.number + 5, difficulty = 104, parent = parent.header.hash)
180+
val newWeight5 = newWeight3.increase(newBlock5.header)
181+
182+
blockchain.save(newBlock5, Nil, newWeight5, saveAsBestBlock = true)
183+
184+
val signatures = CheckpointingTestHelpers.createCheckpointSignatures(
185+
Seq(crypto.generateKeyPair(secureRandom)),
186+
newBlock5.hash
187+
)
188+
blockImporter ! NewCheckpoint(newBlock5.hash, signatures)
189+
190+
val checkpointBlock = checkpointBlockGenerator.generate(newBlock5, Checkpoint(signatures))
191+
192+
Thread.sleep(1000)
193+
blockchain.getBestBlock().get shouldEqual checkpointBlock
194+
blockchain.getLatestCheckpointBlockNumber() shouldEqual newBlock5.header.number + 1
195+
}
196+
}

src/it/scala/io/iohk/ethereum/sync/RegularSyncItSpec.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ class RegularSyncItSpec extends FreeSpecBase with Matchers with BeforeAndAfterAl
3737
_ <- peer2.connectToPeers(Set(peer1.node))
3838
_ <- peer2.waitForRegularSyncLoadLastBlock(blockNumber)
3939
} yield {
40-
assert(peer1.bl.getBestBlock().hash == peer2.bl.getBestBlock().hash)
40+
assert(peer1.bl.getBestBlock().get.hash == peer2.bl.getBestBlock().get.hash)
4141
}
4242
}
4343

@@ -52,7 +52,7 @@ class RegularSyncItSpec extends FreeSpecBase with Matchers with BeforeAndAfterAl
5252
_ <- peer2.connectToPeers(Set(peer1.node))
5353
_ <- peer2.waitForRegularSyncLoadLastBlock(blockHeadersPerRequest + 1)
5454
} yield {
55-
assert(peer1.bl.getBestBlock().hash == peer2.bl.getBestBlock().hash)
55+
assert(peer1.bl.getBestBlock().get.hash == peer2.bl.getBestBlock().get.hash)
5656
}
5757
}
5858
}
@@ -72,7 +72,7 @@ class RegularSyncItSpec extends FreeSpecBase with Matchers with BeforeAndAfterAl
7272
_ <- peer1.mineNewBlocks(100.milliseconds, 2)(IdentityUpdate)
7373
_ <- peer2.waitForRegularSyncLoadLastBlock(blockNumer + 4)
7474
} yield {
75-
assert(peer1.bl.getBestBlock().hash == peer2.bl.getBestBlock().hash)
75+
assert(peer1.bl.getBestBlock().get.hash == peer2.bl.getBestBlock().get.hash)
7676
}
7777
}
7878

@@ -94,8 +94,8 @@ class RegularSyncItSpec extends FreeSpecBase with Matchers with BeforeAndAfterAl
9494
_ <- peer2.waitForRegularSyncLoadLastBlock(blockNumer + 3)
9595
} yield {
9696
assert(
97-
peer1.bl.getChainWeightByHash(peer1.bl.getBestBlock().hash) == peer2.bl.getChainWeightByHash(
98-
peer2.bl.getBestBlock().hash
97+
peer1.bl.getChainWeightByHash(peer1.bl.getBestBlock().get.hash) == peer2.bl.getChainWeightByHash(
98+
peer2.bl.getBestBlock().get.hash
9999
)
100100
)
101101
(peer1.bl.getBlockByNumber(blockNumer + 1), peer2.bl.getBlockByNumber(blockNumer + 1)) match {

src/it/scala/io/iohk/ethereum/sync/util/CommonFakePeer.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
246246
}
247247

248248
def getCurrentState(): BlockchainState = {
249-
val bestBlock = bl.getBestBlock()
249+
val bestBlock = bl.getBestBlock().get
250250
val currentWorldState = getMptForBlock(bestBlock)
251251
val currentWeight = bl.getChainWeightByHash(bestBlock.hash).get
252252
BlockchainState(bestBlock, currentWorldState, currentWeight)
@@ -301,13 +301,13 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
301301
n: BigInt
302302
)(updateWorldForBlock: (BigInt, InMemoryWorldStateProxy) => InMemoryWorldStateProxy): Task[Unit] = {
303303
Task(bl.getBestBlock()).flatMap { block =>
304-
if (block.number >= n) {
304+
if (block.get.number >= n) {
305305
Task(())
306306
} else {
307307
Task {
308-
val currentWeight = bl.getChainWeightByHash(block.hash).get
309-
val currentWolrd = getMptForBlock(block)
310-
val (newBlock, newWeight, _) = createChildBlock(block, currentWeight, currentWolrd)(updateWorldForBlock)
308+
val currentWeight = bl.getChainWeightByHash(block.get.hash).get
309+
val currentWolrd = getMptForBlock(block.get)
310+
val (newBlock, newWeight, _) = createChildBlock(block.get, currentWeight, currentWolrd)(updateWorldForBlock)
311311
bl.save(newBlock, Seq(), newWeight, saveAsBestBlock = true)
312312
broadcastBlock(newBlock, newWeight)
313313
}.flatMap(_ => importBlocksUntil(n)(updateWorldForBlock))

0 commit comments

Comments
 (0)