Merge branch 'develop' into fix/ETCM-927_ChainAtoChainB_difficultyB

Author: strauss-m

nix/overlay.nix

@@ -3,7 +3,7 @@ rev: final: prev: {
 
   mantis = final.callPackage ./mantis.nix {
     src = ../.;
-    depsSha256 = "sha256-US4L/xh2otnEfOa05bazb14bgYhQZpF4GfFY30sDkNY=";
+    depsSha256 = "sha256-QXIsF46M9gKhLD8EWMsyxDRWQNzgzWv3pctRJEEHbYM=";
   };
 
   mantis-hash = final.mantis.override {

project/Dependencies.scala

@@ -70,11 +70,11 @@ object Dependencies {
   )
 
   val cats: Seq[ModuleID] = {
-    val catsVersion = "2.2.0"
+    val catsVersion = "2.6.1"
     Seq(
       "org.typelevel" %% "mouse" % "0.25",
       "org.typelevel" %% "cats-core" % catsVersion,
-      "org.typelevel" %% "cats-effect" % catsVersion
+      "org.typelevel" %% "cats-effect" % "2.5.1"
     )
   }
 

src/it/scala/io/iohk/ethereum/sync/util/CommonFakePeer.scala

@@ -116,6 +116,7 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
   )
 
   val bl = BlockchainImpl(storagesInstance.storages)
+  val evmCodeStorage = storagesInstance.storages.evmCodeStorage
 
   val genesis = Block(
     Fixtures.Blocks.Genesis.header.copy(stateRoot = ByteString(MerklePatriciaTrie.EmptyRootHash)),
@@ -203,7 +204,10 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
   )
 
   val blockchainHost: ActorRef =
-    system.actorOf(BlockchainHostActor.props(bl, peerConf, peerEventBus, etcPeerManager), "blockchain-host")
+    system.actorOf(
+      BlockchainHostActor.props(bl, storagesInstance.storages.evmCodeStorage, peerConf, peerEventBus, etcPeerManager),
+      "blockchain-host"
+    )
 
   lazy val server: ActorRef = system.actorOf(ServerActor.props(nodeStatusHolder, peerManager), "server")
 

src/it/scala/io/iohk/ethereum/sync/util/FastSyncItSpecUtils.scala

@@ -29,6 +29,8 @@ object FastSyncItSpecUtils {
         storagesInstance.storages.fastSyncStateStorage,
         storagesInstance.storages.appStateStorage,
         bl,
+        storagesInstance.storages.evmCodeStorage,
+        storagesInstance.storages.nodeStorage,
         validators,
         peerEventBus,
         etcPeerManager,
@@ -72,7 +74,7 @@ object FastSyncItSpecUtils {
           val codeHash = kec256(accountExpectedCode)
           val accountExpectedStorageAddresses = (i until i + 20).toList
           val account = bl.getAccount(accountAddress, blockNumber).get
-          val code = bl.getEvmCodeByHash(codeHash).get
+          val code = evmCodeStorage.get(codeHash).get
           val storedData = accountExpectedStorageAddresses.map { addr =>
             ByteUtils.toBigInt(bl.getAccountStorageAt(account.storageRoot, addr, ethCompatibleStorage = true))
           }

src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala

@@ -180,8 +180,6 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
 
   override def getChainWeightByHash(blockhash: ByteString): Option[ChainWeight] = ???
 
-  override def getEvmCodeByHash(hash: ByteString): Option[ByteString] = ???
-
   override def getReceiptsByHash(blockhash: ByteString): Option[Seq[Receipt]] = ???
 
   def getAccount(address: Address, blockNumber: BigInt): Option[Account] = ???
@@ -219,6 +217,4 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
   override def save(block: Block, receipts: Seq[Receipt], weight: ChainWeight, saveAsBestBlock: Boolean): Unit = ???
 
   override def getLatestCheckpointBlockNumber(): BigInt = ???
-
-  override def mptStateSavedKeys(): Observable[Either[RocksDbDataSource.IterationError, NodeHash]] = ???
 }

src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala

@@ -46,10 +46,10 @@ object FixtureProvider {
       override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(dataSource)
       override val chainWeightStorage: ChainWeightStorage = new ChainWeightStorage(dataSource)
       override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(dataSource)
-      override val nodeStorage: NodeStorage = new NodeStorage(dataSource)
-      override val cachedNodeStorage: CachedNodeStorage = new CachedNodeStorage(nodeStorage, caches.nodeCache)
-      override val pruningMode: PruningMode = ArchivePruning
       override val appStateStorage: AppStateStorage = new AppStateStorage(dataSource)
+      val nodeStorage: NodeStorage = new NodeStorage(dataSource)
+      val cachedNodeStorage: CachedNodeStorage = new CachedNodeStorage(nodeStorage, caches.nodeCache)
+      val pruningMode: PruningMode = ArchivePruning
       override val stateStorage: StateStorage =
         StateStorage(
           pruningMode,

src/main/resources/conf/chains/etc-chain.conf

@@ -126,6 +126,9 @@
 
     # List of accounts to be drained
     drain-list = null
+
+    # Tells whether this fork should be included on the fork id list used for peer validation
+    include-on-fork-id-list = false
   }
 
   # Starting nonce of an empty account. Some networks (like Morden) use different values.

src/main/resources/conf/chains/eth-chain.conf

@@ -236,6 +236,9 @@
       "bb9bc244d798123fde783fcc1c72d3bb8c189413",
       "807640a13483f8ac783c557fcdf27be11ea4ac7a"
     ]
+
+    # Tells whether this fork should be included on the fork id list used for peer validation
+    include-on-fork-id-list = true
   }
 
   # Starting nonce of an empty account. Some networks (like Morden) use different values.
@@ -292,4 +295,3 @@
     "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303",
   ]
 }
-

src/main/resources/conf/chains/ropsten-chain.conf

@@ -122,6 +122,9 @@
 
     # List of accounts to be drained
     drain-list = null
+
+    # Tells whether this fork should be included on the fork id list used for peer validation
+    include-on-fork-id-list = true
   }
 
   # Starting nonce of an empty account. Some networks (like Morden) use different values.

src/main/resources/conf/chains/test-chain.conf

@@ -119,6 +119,9 @@
 
     # List of accounts to be drained
     drain-list = null
+
+    # Tells whether this fork should be included on the fork id list used for peer validation
+    include-on-fork-id-list = true
   }
 
   # Starting nonce of an empty account. Some networks (like Morden) use different values.

src/main/scala/io/iohk/ethereum/blockchain/sync/BlockchainHostActor.scala

@@ -2,6 +2,7 @@ package io.iohk.ethereum.blockchain.sync
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import akka.util.ByteString
+import io.iohk.ethereum.db.storage.EvmCodeStorage
 import io.iohk.ethereum.domain.{BlockHeader, Blockchain}
 import io.iohk.ethereum.network.PeerEventBusActor.PeerEvent.MessageFromPeer
 import io.iohk.ethereum.network.PeerEventBusActor.SubscriptionClassifier.MessageClassifier
@@ -20,6 +21,7 @@ import io.iohk.ethereum.network.p2p.messages.Codes
   */
 class BlockchainHostActor(
     blockchain: Blockchain,
+    evmCodeStorage: EvmCodeStorage,
     peerConfiguration: PeerConfiguration,
     peerEventBusActor: ActorRef,
     etcPeerManagerActor: ActorRef
@@ -54,7 +56,7 @@ class BlockchainHostActor(
         val maybeMptNodeData = blockchain.getMptNodeByHash(hash).map(e => e.toBytes: ByteString)
 
         //If no mpt node was found, fetch evm by hash
-        maybeMptNodeData.orElse(blockchain.getEvmCodeByHash(hash))
+        maybeMptNodeData.orElse(evmCodeStorage.get(hash))
       }
 
       Some(NodeData(nodeData))
@@ -116,10 +118,13 @@ object BlockchainHostActor {
 
   def props(
       blockchain: Blockchain,
+      evmCodeStorage: EvmCodeStorage,
       peerConfiguration: PeerConfiguration,
       peerEventBusActor: ActorRef,
       etcPeerManagerActor: ActorRef
   ): Props =
-    Props(new BlockchainHostActor(blockchain, peerConfiguration, peerEventBusActor, etcPeerManagerActor))
+    Props(
+      new BlockchainHostActor(blockchain, evmCodeStorage, peerConfiguration, peerEventBusActor, etcPeerManagerActor)
+    )
 
 }

src/main/scala/io/iohk/ethereum/blockchain/sync/SyncController.scala

@@ -4,14 +4,16 @@ import akka.actor.{Actor, ActorLogging, ActorRef, PoisonPill, Props, Scheduler}
 import io.iohk.ethereum.blockchain.sync.fast.FastSync
 import io.iohk.ethereum.blockchain.sync.regular.RegularSync
 import io.iohk.ethereum.consensus.validators.Validators
-import io.iohk.ethereum.db.storage.{AppStateStorage, FastSyncStateStorage}
+import io.iohk.ethereum.db.storage.{AppStateStorage, EvmCodeStorage, FastSyncStateStorage, NodeStorage}
 import io.iohk.ethereum.domain.Blockchain
 import io.iohk.ethereum.ledger.Ledger
 import io.iohk.ethereum.utils.Config.SyncConfig
 
 class SyncController(
     appStateStorage: AppStateStorage,
     blockchain: Blockchain,
+    evmCodeStorage: EvmCodeStorage,
+    nodeStorage: NodeStorage,
     fastSyncStateStorage: FastSyncStateStorage,
     ledger: Ledger,
     validators: Validators,
@@ -77,6 +79,8 @@ class SyncController(
         fastSyncStateStorage,
         appStateStorage,
         blockchain,
+        evmCodeStorage,
+        nodeStorage,
         validators,
         peerEventBus,
         etcPeerManager,
@@ -120,6 +124,8 @@ object SyncController {
   def props(
       appStateStorage: AppStateStorage,
       blockchain: Blockchain,
+      evmCodeStorage: EvmCodeStorage,
+      nodeStorage: NodeStorage,
       syncStateStorage: FastSyncStateStorage,
       ledger: Ledger,
       validators: Validators,
@@ -134,6 +140,8 @@ object SyncController {
       new SyncController(
         appStateStorage,
         blockchain,
+        evmCodeStorage,
+        nodeStorage,
         syncStateStorage,
         ledger,
         validators,

src/main/scala/io/iohk/ethereum/blockchain/sync/fast/FastSync.scala

@@ -20,7 +20,7 @@ import io.iohk.ethereum.blockchain.sync.fast.SyncStateSchedulerActor.{
   WaitingForNewTargetBlock
 }
 import io.iohk.ethereum.consensus.validators.Validators
-import io.iohk.ethereum.db.storage.{AppStateStorage, FastSyncStateStorage}
+import io.iohk.ethereum.db.storage.{AppStateStorage, EvmCodeStorage, FastSyncStateStorage, NodeStorage}
 import io.iohk.ethereum.domain._
 import io.iohk.ethereum.mpt.MerklePatriciaTrie
 import io.iohk.ethereum.network.EtcPeerManagerActor.PeerInfo
@@ -48,6 +48,8 @@ class FastSync(
     val fastSyncStateStorage: FastSyncStateStorage,
     val appStateStorage: AppStateStorage,
     val blockchain: Blockchain,
+    evmCodeStorage: EvmCodeStorage,
+    nodeStorage: NodeStorage,
     val validators: Validators,
     val peerEventBus: ActorRef,
     val etcPeerManager: ActorRef,
@@ -153,7 +155,7 @@ class FastSync(
     private val syncStateScheduler = context.actorOf(
       SyncStateSchedulerActor
         .props(
-          SyncStateScheduler(blockchain, syncConfig.stateSyncBloomFilterSize),
+          SyncStateScheduler(blockchain, evmCodeStorage, nodeStorage, syncConfig.stateSyncBloomFilterSize),
           syncConfig,
           etcPeerManager,
           peerEventBus,
@@ -1134,6 +1136,8 @@ object FastSync {
       fastSyncStateStorage: FastSyncStateStorage,
       appStateStorage: AppStateStorage,
       blockchain: Blockchain,
+      evmCodeStorage: EvmCodeStorage,
+      nodeStorage: NodeStorage,
       validators: Validators,
       peerEventBus: ActorRef,
       etcPeerManager: ActorRef,
@@ -1146,6 +1150,8 @@ object FastSync {
         fastSyncStateStorage,
         appStateStorage,
         blockchain,
+        evmCodeStorage,
+        nodeStorage,
         validators,
         peerEventBus,
         etcPeerManager,

src/main/scala/io/iohk/ethereum/blockchain/sync/fast/SyncStateScheduler.scala

@@ -1,16 +1,18 @@
 package io.iohk.ethereum.blockchain.sync.fast
 
 import java.util.Comparator
-
 import akka.util.ByteString
 import com.google.common.hash.{BloomFilter, Funnel, PrimitiveSink}
 import io.iohk.ethereum.blockchain.sync.fast.LoadableBloomFilter.BloomFilterLoadingResult
 import io.iohk.ethereum.blockchain.sync.fast.SyncStateScheduler._
+import io.iohk.ethereum.db.dataSource.RocksDbDataSource.IterationError
+import io.iohk.ethereum.db.storage.{EvmCodeStorage, NodeStorage}
 import io.iohk.ethereum.domain.{Account, Blockchain}
 import io.iohk.ethereum.mpt.{BranchNode, ExtensionNode, HashNode, LeafNode, MerklePatriciaTrie, MptNode}
 import io.iohk.ethereum.network.p2p.messages.ETH63.MptNodeEncoders.MptNodeDec
 import io.vavr.collection.PriorityQueue
 import monix.eval.Task
+import monix.reactive.Observable
 
 import scala.annotation.tailrec
 import scala.collection.immutable.ArraySeq
@@ -43,7 +45,11 @@ import scala.util.Try
   *
   * Important part is that nodes retrieved by getMissingNodes, must eventually be provided for scheduler to make progress
   */
-class SyncStateScheduler(blockchain: Blockchain, bloomFilter: LoadableBloomFilter[ByteString]) {
+class SyncStateScheduler(
+    blockchain: Blockchain,
+    evmCodeStorage: EvmCodeStorage,
+    bloomFilter: LoadableBloomFilter[ByteString]
+) {
 
   val loadFilterFromBlockchain: Task[BloomFilterLoadingResult] = bloomFilter.loadFromSource
 
@@ -242,9 +248,9 @@ class SyncStateScheduler(blockchain: Blockchain, bloomFilter: LoadableBloomFilte
 
   private def isInDatabase(req: StateNodeRequest): Boolean = {
     req.requestType match {
-      case request: CodeRequest =>
-        blockchain.getEvmCodeByHash(req.nodeHash).isDefined
-      case request: NodeRequest =>
+      case _: CodeRequest =>
+        evmCodeStorage.get(req.nodeHash).isDefined
+      case _: NodeRequest =>
         blockchain.getMptNodeByHash(req.nodeHash).isDefined
     }
   }
@@ -284,12 +290,22 @@ object SyncStateScheduler {
     BloomFilter.create[ByteString](ByteStringFunnel, expectedFilterSize)
   }
 
-  def apply(blockchain: Blockchain, expectedBloomFilterSize: Int): SyncStateScheduler = {
+  def apply(
+      blockchain: Blockchain,
+      evmCodeStorage: EvmCodeStorage,
+      nodeStorage: NodeStorage,
+      expectedBloomFilterSize: Int
+  ): SyncStateScheduler = {
     // provided source i.e mptStateSavedKeys() is guaranteed to finish on first `Left` element which means that returned
     // error is the reason why loading has stopped
+    val mptStateSavedKeys: Observable[Either[IterationError, ByteString]] =
+      (nodeStorage.storageContent.map(c => c.map(_._1)) ++ evmCodeStorage.storageContent.map(c => c.map(_._1)))
+        .takeWhileInclusive(_.isRight)
+
     new SyncStateScheduler(
       blockchain,
-      LoadableBloomFilter[ByteString](expectedBloomFilterSize, blockchain.mptStateSavedKeys())
+      evmCodeStorage,
+      LoadableBloomFilter[ByteString](expectedBloomFilterSize, mptStateSavedKeys)
     )
   }