Skip to content

Commit ee341ce

Browse files
rtkaczykpslaski
rtkaczyk
authored and
pslaski
committed
[ETCM-263] improve BlockImport wrt checkpoints
1 parent 5db9fb9 commit ee341ce

File tree

75 files changed

+937
-885
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+937
-885
lines changed

‎src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,12 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
8585
Block(scenario.genesisBlockHeader.toBlockHeader, BlockBody(Nil, Nil))
8686
}
8787

88+
val genesisWeight = ChainWeight.zero.increase(genesisBlock.header)
89+
8890
blockchain
8991
.storeBlock(genesisBlock)
90-
.and(blockchain.storeReceipts(genesisBlock.header.hash, Nil))
91-
.and(blockchain.storeTotalDifficulty(genesisBlock.header.hash, genesisBlock.header.difficulty))
92+
.and(blockchain.storeReceipts(genesisBlock.hash, Nil))
93+
.and(blockchain.storeChainWeight(genesisBlock.hash, genesisWeight))
9294
.commit()
9395

9496
genesisBlock

‎src/it/scala/io/iohk/ethereum/sync/RegularSyncItSpec.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,13 @@ class RegularSyncItSpec extends FreeSpecBase with Matchers with BeforeAndAfterAl
8484
_ <- peer2.waitForRegularSyncLoadLastBlock(blockNumer + 3)
8585
} yield {
8686
assert(
87-
peer1.bl.getTotalDifficultyByHash(peer1.bl.getBestBlock().hash) == peer2.bl.getTotalDifficultyByHash(
87+
peer1.bl.getChainWeightByHash(peer1.bl.getBestBlock().hash) == peer2.bl.getChainWeightByHash(
8888
peer2.bl.getBestBlock().hash
8989
)
9090
)
9191
(peer1.bl.getBlockByNumber(blockNumer + 1), peer2.bl.getBlockByNumber(blockNumer + 1)) match {
9292
case (Some(blockP1), Some(blockP2)) =>
93-
assert(peer1.bl.getTotalDifficultyByHash(blockP1.hash) == peer2.bl.getTotalDifficultyByHash(blockP2.hash))
93+
assert(peer1.bl.getChainWeightByHash(blockP1.hash) == peer2.bl.getChainWeightByHash(blockP2.hash))
9494
case (_, _) => fail("invalid difficulty validation")
9595
}
9696
}

‎src/it/scala/io/iohk/ethereum/sync/util/CommonFakePeer.scala

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import io.iohk.ethereum.db.components.{RocksDbDataSourceComponent, Storages}
1313
import io.iohk.ethereum.db.dataSource.{RocksDbConfig, RocksDbDataSource}
1414
import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
1515
import io.iohk.ethereum.db.storage.{AppStateStorage, Namespaces}
16-
import io.iohk.ethereum.domain.{Block, Blockchain, BlockchainImpl}
16+
import io.iohk.ethereum.domain.{Block, Blockchain, BlockchainImpl, ChainWeight}
1717
import io.iohk.ethereum.ledger.InMemoryWorldStateProxy
1818
import io.iohk.ethereum.mpt.MerklePatriciaTrie
1919
import io.iohk.ethereum.network.EtcPeerManagerActor.PeerInfo
@@ -116,8 +116,9 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
116116
Fixtures.Blocks.Genesis.header.copy(stateRoot = ByteString(MerklePatriciaTrie.EmptyRootHash)),
117117
Fixtures.Blocks.Genesis.body
118118
)
119+
val genesisWeight = ChainWeight.zero.increase(genesis.header)
119120

120-
bl.save(genesis, Seq(), genesis.header.difficulty, saveAsBestBlock = true)
121+
bl.save(genesis, Seq(), genesisWeight, saveAsBestBlock = true)
121122

122123
lazy val nh = nodeStatusHolder
123124

@@ -228,15 +229,15 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
228229
)
229230
}
230231

231-
private def broadcastBlock(block: Block, td: BigInt) = {
232-
broadcasterActor ! BroadcastBlock(NewBlock(block, td))
232+
private def broadcastBlock(block: Block, weight: ChainWeight) = {
233+
broadcasterActor ! BroadcastBlock(NewBlock(block, weight))
233234
}
234235

235236
def getCurrentState(): BlockchainState = {
236237
val bestBlock = bl.getBestBlock()
237238
val currentWorldState = getMptForBlock(bestBlock)
238-
val currentTd = bl.getTotalDifficultyByHash(bestBlock.hash).get
239-
BlockchainState(bestBlock, currentWorldState, currentTd)
239+
val currentWeight = bl.getChainWeightByHash(bestBlock.hash).get
240+
BlockchainState(bestBlock, currentWorldState, currentWeight)
240241
}
241242

242243
def startPeer(): Task[Unit] = {
@@ -272,16 +273,16 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
272273
} yield ()
273274
}
274275

275-
private def createChildBlock(parent: Block, parentTd: BigInt, parentWorld: InMemoryWorldStateProxy)(
276+
private def createChildBlock(parent: Block, parentWeight: ChainWeight, parentWorld: InMemoryWorldStateProxy)(
276277
updateWorldForBlock: (BigInt, InMemoryWorldStateProxy) => InMemoryWorldStateProxy
277-
): (Block, BigInt, InMemoryWorldStateProxy) = {
278+
): (Block, ChainWeight, InMemoryWorldStateProxy) = {
278279
val newBlockNumber = parent.header.number + 1
279280
val newWorld = updateWorldForBlock(newBlockNumber, parentWorld)
280281
val newBlock = parent.copy(header =
281282
parent.header.copy(parentHash = parent.header.hash, number = newBlockNumber, stateRoot = newWorld.stateRootHash)
282283
)
283-
val newTd = newBlock.header.difficulty + parentTd
284-
(newBlock, newTd, parentWorld)
284+
val newWeight = parentWeight.increase(newBlock.header)
285+
(newBlock, newWeight, parentWorld)
285286
}
286287

287288
def importBlocksUntil(
@@ -292,12 +293,12 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
292293
Task(())
293294
} else {
294295
Task {
295-
val currentTd = bl.getTotalDifficultyByHash(block.hash).get
296+
val currentWeight = bl.getChainWeightByHash(block.hash).get
296297
val currentWolrd = getMptForBlock(block)
297-
val (newBlock, newTd, newWorld) = createChildBlock(block, currentTd, currentWolrd)(updateWorldForBlock)
298-
bl.save(newBlock, Seq(), newTd, saveAsBestBlock = true)
298+
val (newBlock, newWeight, _) = createChildBlock(block, currentWeight, currentWolrd)(updateWorldForBlock)
299+
bl.save(newBlock, Seq(), newWeight, saveAsBestBlock = true)
299300
bl.persistCachedNodes()
300-
broadcastBlock(newBlock, newTd)
301+
broadcastBlock(newBlock, newWeight)
301302
}.flatMap(_ => importBlocksUntil(n)(updateWorldForBlock))
302303
}
303304
}

‎src/it/scala/io/iohk/ethereum/sync/util/RegularSyncItSpecUtils.scala

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -92,12 +92,12 @@ object RegularSyncItSpecUtils {
9292
case None => bl.getBestBlock()
9393
}).flatMap { block =>
9494
Task {
95-
val currentTd = bl
96-
.getTotalDifficultyByHash(block.hash)
97-
.getOrElse(throw new RuntimeException(s"block by hash: ${block.hash} doesn't exist"))
98-
val currentWolrd = getMptForBlock(block)
99-
val (newBlock, newTd, newWorld) = createChildBlock(block, currentTd, currentWolrd)(updateWorldForBlock)
100-
broadcastBlock(newBlock, newTd)
95+
val currentWeight = bl
96+
.getChainWeightByHash(block.hash)
97+
.getOrElse(throw new RuntimeException(s"ChainWeight by hash: ${block.hash} doesn't exist"))
98+
val currentWorld = getMptForBlock(block)
99+
val (newBlock, newWeight, _) = createChildBlock(block, currentWeight, currentWorld)(updateWorldForBlock)
100+
broadcastBlock(newBlock, newWeight)
101101
}
102102
}
103103
}
@@ -110,12 +110,12 @@ object RegularSyncItSpecUtils {
110110
plusDifficulty: BigInt = 0
111111
)(updateWorldForBlock: (BigInt, InMemoryWorldStateProxy) => InMemoryWorldStateProxy): Task[Unit] = Task {
112112
val block: Block = bl.getBestBlock()
113-
val currentTd = bl
114-
.getTotalDifficultyByHash(block.hash)
115-
.getOrElse(throw new RuntimeException(s"block by hash: ${block.hash} doesn't exist"))
113+
val currentWeight = bl
114+
.getChainWeightByHash(block.hash)
115+
.getOrElse(throw new RuntimeException(s"ChainWeight by hash: ${block.hash} doesn't exist"))
116116
val currentWolrd = getMptForBlock(block)
117-
val (newBlock, newTd, newWorld) =
118-
createChildBlock(block, currentTd, currentWolrd, plusDifficulty)(updateWorldForBlock)
117+
val (newBlock, _, _) =
118+
createChildBlock(block, currentWeight, currentWolrd, plusDifficulty)(updateWorldForBlock)
119119
regularSync ! SyncProtocol.MinedBlock(newBlock)
120120
}
121121

@@ -139,18 +139,18 @@ object RegularSyncItSpecUtils {
139139
)
140140
}
141141

142-
private def broadcastBlock(block: Block, td: BigInt) = {
143-
broadcasterActor ! BroadcastBlock(NewBlock(block, td))
142+
private def broadcastBlock(block: Block, weight: ChainWeight) = {
143+
broadcasterActor ! BroadcastBlock(NewBlock(block, weight))
144144
}
145145

146146
private def createChildBlock(
147147
parent: Block,
148-
parentTd: BigInt,
148+
parentWeight: ChainWeight,
149149
parentWorld: InMemoryWorldStateProxy,
150150
plusDifficulty: BigInt = 0
151151
)(
152152
updateWorldForBlock: (BigInt, InMemoryWorldStateProxy) => InMemoryWorldStateProxy
153-
): (Block, BigInt, InMemoryWorldStateProxy) = {
153+
): (Block, ChainWeight, InMemoryWorldStateProxy) = {
154154
val newBlockNumber = parent.header.number + 1
155155
val newWorld = updateWorldForBlock(newBlockNumber, parentWorld)
156156
val newBlock = parent.copy(header =
@@ -161,8 +161,8 @@ object RegularSyncItSpecUtils {
161161
difficulty = plusDifficulty + parent.header.difficulty
162162
)
163163
)
164-
val newTd = newBlock.header.difficulty + parentTd
165-
(newBlock, newTd, parentWorld)
164+
val newWeight = parentWeight.increase(newBlock.header)
165+
(newBlock, newWeight, parentWorld)
166166
}
167167
}
168168

‎src/it/scala/io/iohk/ethereum/sync/util/SyncCommonItSpec.scala

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package io.iohk.ethereum.sync.util
22

33
import java.net.{InetSocketAddress, ServerSocket}
44

5-
import io.iohk.ethereum.domain.Block
5+
import io.iohk.ethereum.domain.{Block, ChainWeight}
66
import io.iohk.ethereum.ledger.InMemoryWorldStateProxy
77

88
object SyncCommonItSpec {
@@ -17,5 +17,9 @@ object SyncCommonItSpec {
1717
}
1818
}
1919

20-
final case class BlockchainState(bestBlock: Block, currentWorldState: InMemoryWorldStateProxy, currentTd: BigInt)
20+
final case class BlockchainState(
21+
bestBlock: Block,
22+
currentWorldState: InMemoryWorldStateProxy,
23+
currentWeight: ChainWeight
24+
)
2125
}

‎src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,13 +148,13 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
148148

149149
override def storeEvmCode(hash: ByteString, evmCode: ByteString): DataSourceBatchUpdate = ???
150150

151-
override def storeTotalDifficulty(blockhash: ByteString, totalDifficulty: BigInt): DataSourceBatchUpdate = ???
151+
override def storeChainWeight(blockhash: ByteString, chainWeight: ChainWeight): DataSourceBatchUpdate = ???
152152

153153
override def saveNode(nodeHash: NodeHash, nodeEncoded: NodeEncoded, blockNumber: BigInt): Unit = ???
154154

155155
override def removeBlock(hash: ByteString, withState: Boolean = true): Unit = ???
156156

157-
override def getTotalDifficultyByHash(blockhash: ByteString): Option[BigInt] = ???
157+
override def getChainWeightByHash(blockhash: ByteString): Option[ChainWeight] = ???
158158

159159
override def getEvmCodeByHash(hash: ByteString): Option[ByteString] = ???
160160

@@ -194,7 +194,7 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
194194

195195
def getBestBlock(): Block = ???
196196

197-
override def save(block: Block, receipts: Seq[Receipt], totalDifficulty: BigInt, saveAsBestBlock: Boolean): Unit = ???
197+
override def save(block: Block, receipts: Seq[Receipt], weight: ChainWeight, saveAsBestBlock: Boolean): Unit = ???
198198

199199
override def getStateStorage: StateStorage = ???
200200

‎src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ object FixtureProvider {
4444
override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(dataSource)
4545
override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(dataSource)
4646
override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(dataSource)
47-
override val totalDifficultyStorage: TotalDifficultyStorage = new TotalDifficultyStorage(dataSource)
47+
override val chainWeightStorage: ChainWeightStorage = new ChainWeightStorage(dataSource)
4848
override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(dataSource)
4949
override val nodeStorage: NodeStorage = new NodeStorage(dataSource)
5050
override val cachedNodeStorage: CachedNodeStorage = new CachedNodeStorage(nodeStorage, caches.nodeCache)

‎src/main/scala/io/iohk/ethereum/blockchain/data/GenesisDataLoader.scala

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,12 @@ class GenesisDataLoader(blockchain: Blockchain, blockchainConfig: BlockchainConf
118118
case None =>
119119
storage.persist()
120120
stateStorage.forcePersist(GenesisDataLoad)
121-
blockchain.save(Block(header, BlockBody(Nil, Nil)), Nil, header.difficulty, saveAsBestBlock = true)
121+
blockchain.save(
122+
Block(header, BlockBody(Nil, Nil)),
123+
Nil,
124+
ChainWeight.totalDifficultyOnly(header.difficulty),
125+
saveAsBestBlock = true
126+
)
122127
Success(())
123128
}
124129
}

‎src/main/scala/io/iohk/ethereum/blockchain/sync/BlockBroadcast.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,7 @@ class BlockBroadcast(val etcPeerManager: ActorRef, syncConfig: SyncConfig) {
3636

3737
private def shouldSendNewBlock(newBlock: NewBlock, peerInfo: PeerInfo): Boolean =
3838
newBlock.block.header.number > peerInfo.maxBlockNumber ||
39-
newBlock.totalDifficulty > peerInfo.totalDifficulty ||
40-
newBlock.latestCheckpointNumber > peerInfo.latestCheckpointNumber
39+
newBlock.chainWeight > peerInfo.chainWeight
4140

4241
private def broadcastNewBlock(newBlock: NewBlock, peers: Set[Peer]): Unit =
4342
obtainRandomPeerSubset(peers).foreach { peer =>

‎src/main/scala/io/iohk/ethereum/blockchain/sync/FastSync.scala

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -341,8 +341,8 @@ class FastSync(
341341
val header = headers.head
342342
processHeader(header, peer) match {
343343
case Left(result) => result
344-
case Right(headerAndDif) =>
345-
updateSyncState(headerAndDif._1, headerAndDif._2)
344+
case Right((header, weight)) =>
345+
updateSyncState(header, weight)
346346
if (header.number == syncState.safeDownloadTarget) {
347347
ImportedPivotBlock
348348
} else {
@@ -371,10 +371,10 @@ class FastSync(
371371
}
372372
}
373373

374-
private def updateSyncState(header: BlockHeader, parentTd: BigInt): Unit = {
374+
private def updateSyncState(header: BlockHeader, parentWeight: ChainWeight): Unit = {
375375
blockchain
376376
.storeBlockHeader(header)
377-
.and(blockchain.storeTotalDifficulty(header.hash, parentTd + header.difficulty))
377+
.and(blockchain.storeChainWeight(header.hash, parentWeight.increase(header)))
378378
.commit()
379379

380380
if (header.number > syncState.bestBlockHeaderNumber) {
@@ -391,14 +391,17 @@ class FastSync(
391391
syncState = syncState.updateNextBlockToValidate(header, K, X)
392392
}
393393

394-
private def processHeader(header: BlockHeader, peer: Peer): Either[HeaderProcessingResult, (BlockHeader, BigInt)] =
394+
private def processHeader(
395+
header: BlockHeader,
396+
peer: Peer
397+
): Either[HeaderProcessingResult, (BlockHeader, ChainWeight)] =
395398
for {
396399
validatedHeader <- validateHeader(header, peer)
397-
parentDifficulty <- getParentDifficulty(header)
398-
} yield (validatedHeader, parentDifficulty)
400+
parentWeight <- getParentChainWeight(header)
401+
} yield (validatedHeader, parentWeight)
399402

400-
private def getParentDifficulty(header: BlockHeader) = {
401-
blockchain.getTotalDifficultyByHash(header.parentHash).toRight(ParentDifficultyNotFound(header))
403+
private def getParentChainWeight(header: BlockHeader) = {
404+
blockchain.getChainWeightByHash(header.parentHash).toRight(ParentChainWeightNotFound(header))
402405
}
403406

404407
private def handleRewind(header: BlockHeader, peer: Peer, N: Int): Unit = {
@@ -419,11 +422,11 @@ class FastSync(
419422
private def handleBlockHeaders(peer: Peer, headers: Seq[BlockHeader]) = {
420423
if (checkHeadersChain(headers)) {
421424
processHeaders(peer, headers) match {
422-
case ParentDifficultyNotFound(header) =>
425+
case ParentChainWeightNotFound(header) =>
423426
// We could end in wrong fork and get blocked so we should rewind our state a little
424427
// we blacklist peer just in case we got malicious peer which would send us bad blocks, forcing us to rollback
425428
// to genesis
426-
log.warning("Parent difficulty not found for block {}, not processing rest of headers", header.idTag)
429+
log.warning("Parent chain weight not found for block {}, not processing rest of headers", header.idTag)
427430
handleRewind(header, peer, syncConfig.fastSyncBlockValidationN)
428431
case HeadersProcessingFinished =>
429432
processSyncing()
@@ -890,7 +893,7 @@ object FastSync {
890893

891894
sealed abstract class HeaderProcessingResult
892895
case object HeadersProcessingFinished extends HeaderProcessingResult
893-
case class ParentDifficultyNotFound(header: BlockHeader) extends HeaderProcessingResult
896+
case class ParentChainWeightNotFound(header: BlockHeader) extends HeaderProcessingResult
894897
case class ValidationFailed(header: BlockHeader, peer: Peer) extends HeaderProcessingResult
895898
case object ImportedPivotBlock extends HeaderProcessingResult
896899

‎src/main/scala/io/iohk/ethereum/blockchain/sync/PeersClient.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,12 +161,12 @@ object PeersClient {
161161

162162
def bestPeer(peersToDownloadFrom: Map[Peer, PeerInfo]): Option[Peer] = {
163163
val peersToUse = peersToDownloadFrom
164-
.collect { case (ref, PeerInfo(_, totalDifficulty, latestChkp, true, _, _)) =>
165-
(ref, totalDifficulty, latestChkp)
164+
.collect { case (ref, PeerInfo(_, chainWeight, true, _, _)) =>
165+
(ref, chainWeight)
166166
}
167167

168168
if (peersToUse.nonEmpty) {
169-
val (peer, _, _) = peersToUse.maxBy { case (_, td, latestChkp) => latestChkp -> td }
169+
val (peer, _) = peersToUse.maxBy(_._2)
170170
Some(peer)
171171
} else {
172172
None

‎src/main/scala/io/iohk/ethereum/blockchain/sync/PivotBlockSelector.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,8 @@ class PivotBlockSelector(
173173
}
174174

175175
private def collectVoters: ElectionDetails = {
176-
val peersUsedToChooseTarget = peersToDownloadFrom.collect {
177-
case (peer, PeerInfo(_, _, _, true, maxBlockNumber, _)) =>
178-
(peer, maxBlockNumber)
176+
val peersUsedToChooseTarget = peersToDownloadFrom.collect { case (peer, PeerInfo(_, _, true, maxBlockNumber, _)) =>
177+
(peer, maxBlockNumber)
179178
}
180179

181180
val peersSortedByBestNumber = peersUsedToChooseTarget.toList.sortBy { case (_, number) => -number }

‎src/main/scala/io/iohk/ethereum/blockchain/sync/SyncStateSchedulerActor.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ class SyncStateSchedulerActor(
9898
case Some((startSignal: StartSyncingTo, sender)) =>
9999
val initStats = ProcessingStatistics().addSaved(result.writtenElements)
100100
startSyncing(startSignal.stateRoot, startSignal.blockNumber, initStats, sender)
101-
case Some((restartSignal: RestartRequested.type, sender)) =>
101+
case Some((RestartRequested, sender)) =>
102+
// TODO: are we testing this path?
102103
sender ! WaitingForNewTargetBlock
103104
context.become(idle(ProcessingStatistics().addSaved(result.writtenElements)))
104105
case _ =>

‎src/main/scala/io/iohk/ethereum/blockchain/sync/regular/BlockFetcher.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ class BlockFetcher(
192192
supervisor ! ProgressProtocol.GotNewBlock(newState.knownTop)
193193

194194
fetchBlocks(newState)
195-
case MessageFromPeer(NewBlock(block, _, _), peerId) =>
195+
case MessageFromPeer(NewBlock(_, block, _), peerId) =>
196196
val newBlockNr = block.number
197197
val nextExpectedBlock = state.lastFullBlockNumber + 1
198198

0 commit comments

Comments
 (0)