[ETCM-27] Update and check ETS tests

Author: mmrozek

src/ets/resources/ets

@@ -1,16 +1,14 @@
 package io.iohk.ethereum.ets.blockchain
 
 import java.util.concurrent.Executors
-
 import akka.actor.ActorSystem
 import io.iohk.ethereum.domain.Block
 import io.iohk.ethereum.ets.common.TestOptions
 import io.iohk.ethereum.extvm.ExtVMInterface
 import io.iohk.ethereum.ledger.Ledger.VMImpl
 import io.iohk.ethereum.nodebuilder.VmSetup
-import io.iohk.ethereum.utils.{ Config, Logger, VmConfig}
+import io.iohk.ethereum.utils.{Config, Logger, VmConfig}
 import org.scalatest._
-
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, ExecutionContext, Future}
 
@@ -23,7 +21,9 @@ object BlockchainSuite {
 class BlockchainSuite extends FreeSpec with Matchers with BeforeAndAfterAll with Logger {
   import BlockchainSuite.testContext
 
-  val unsupportedNetworks: Set[String] = Set()
+  val unsupportedNetworks: Set[String] = Set(
+    "Berlin"
+  )
   val supportedNetworks = Set(
     "EIP150",
     "Frontier",
@@ -35,7 +35,9 @@ class BlockchainSuite extends FreeSpec with Matchers with BeforeAndAfterAll with
     "Byzantium",
     "EIP158ToByzantiumAt5",
     "Constantinople",
-    "ByzantiumToConstantinopleAt5"
+    "ByzantiumToConstantinopleFixAt5",
+    "ConstantinopleFix",
+    "Istanbul"
   )
   // Map of ignored tests, empty set of ignored names means cancellation of whole group
   val ignoredTests: Map[String, Set[String]] = Map()
@@ -107,12 +109,13 @@ class BlockchainSuite extends FreeSpec with Matchers with BeforeAndAfterAll with
 
     val lastBlock = getBestBlock
 
-    val expectedWorldStateHash = finalWorld.stateRootHash
+    val expectedWorldStateHash =
+      scenario.postStateHash.orElse(finalWorld.map(_.stateRootHash)).getOrElse(throw new IllegalStateException("postState or PostStateHash not defined"))
 
     lastBlock shouldBe defined
 
-    val expectedState = getExpectedState
-    val resultState = getResultState
+    val expectedState = getExpectedState.toList.flatten
+    val resultState = getResultState.toList.flatten
 
     lastBlock.get.header.hash shouldEqual scenario.lastblockhash
     resultState should contain theSameElementsAs expectedState

src/ets/scala/io/iohk/ethereum/ets/blockchain/BlockchainSuite.scala

@@ -19,13 +19,14 @@ trait BlockchainTestConfig extends BlockchainConfig {
   val eip161BlockNumber: BigInt = Long.MaxValue
   val byzantiumBlockNumber: BigInt = Long.MaxValue
   val constantinopleBlockNumber: BigInt = Long.MaxValue
+  val petersburgBlockNumber: BigInt = Long.MaxValue
   val istanbulBlockNumber: BigInt = Long.MaxValue
   // unused
   override val maxCodeSize: Option[BigInt] = None
   override val difficultyBombPauseBlockNumber: BigInt = 3000000
   override val difficultyBombContinueBlockNumber: BigInt = 5000000
   override val difficultyBombRemovalBlockNumber: BigInt = 5900000
-  override val chainId: Byte = 0x3d.toByte
+  override val chainId: Byte = 0x1.toByte
   override val networkId: Int = 1
   override val customGenesisFileOpt: Option[String] = Some("test-genesis.json")
   override val monetaryPolicyConfig: MonetaryPolicyConfig =
@@ -40,7 +41,6 @@ trait BlockchainTestConfig extends BlockchainConfig {
   override val atlantisBlockNumber: BigInt = Long.MaxValue
   override val aghartaBlockNumber: BigInt = Long.MaxValue
   override val phoenixBlockNumber: BigInt = Long.MaxValue
-  override val petersburgBlockNumber: BigInt = Long.MaxValue
 }
 
 object FrontierConfig extends BlockchainTestConfig {
@@ -229,6 +229,35 @@ object ConstantinopleConfig extends BlockchainTestConfig {
   override val monetaryPolicyConfig: MonetaryPolicyConfig =
     MonetaryPolicyConfig(5000000, 0.2, BigInt("5000000000000000000"), BigInt("3000000000000000000"), BigInt("2000000000000000000"))
 }
+object ConstantinopleFixConfig extends BlockchainTestConfig {
+  override val frontierBlockNumber: BigInt = -1
+  override val homesteadBlockNumber: BigInt = -1
+  override val eip150BlockNumber: BigInt = -1
+  override val eip155BlockNumber: BigInt = -1
+  override val eip160BlockNumber: BigInt = -1
+  override val eip161BlockNumber: BigInt = -1
+  override val maxCodeSize: Option[BigInt] = Some(24576)
+  override val byzantiumBlockNumber: BigInt = -1
+  override val constantinopleBlockNumber: BigInt = -1
+  override val petersburgBlockNumber: BigInt = 0
+  override val monetaryPolicyConfig: MonetaryPolicyConfig =
+    MonetaryPolicyConfig(5000000, 0.2, BigInt("5000000000000000000"), BigInt("3000000000000000000"), BigInt("2000000000000000000"))
+}
+object IstanbulConfig extends BlockchainTestConfig {
+  override val frontierBlockNumber: BigInt = -1
+  override val homesteadBlockNumber: BigInt = -1
+  override val eip150BlockNumber: BigInt = -1
+  override val eip155BlockNumber: BigInt = -1
+  override val eip160BlockNumber: BigInt = -1
+  override val eip161BlockNumber: BigInt = -1
+  override val maxCodeSize: Option[BigInt] = Some(24576)
+  override val byzantiumBlockNumber: BigInt = -1
+  override val constantinopleBlockNumber: BigInt = -1
+  override val petersburgBlockNumber: BigInt = -1
+  override val istanbulBlockNumber: BigInt = 0
+  override val monetaryPolicyConfig: MonetaryPolicyConfig =
+    MonetaryPolicyConfig(5000000, 0.2, BigInt("5000000000000000000"), BigInt("3000000000000000000"), BigInt("2000000000000000000"))
+}
 object Eip158ToByzantiumAt5Config extends BlockchainTestConfig {
   override val frontierBlockNumber: BigInt = -1
   override val homesteadBlockNumber: BigInt = -1
@@ -267,6 +296,8 @@ object Validators {
   val eip158Validators = EthashValidators(Eip158Config)
   val byzantiumValidators = EthashValidators(ByzantiumConfig)
   val constantinopleValidators = EthashValidators(ConstantinopleConfig)
+  val constantinopleFixValidators = EthashValidators(ConstantinopleFixConfig)
+  val istanbulValidators = EthashValidators(IstanbulConfig)
   val eip158ToByzantiumValidators = EthashValidators(Eip158ToByzantiumAt5Config)
   val byzantiumToConstantinopleAt5 = EthashValidators(ByzantiumToConstantinopleAt5)
 }
@@ -283,6 +314,8 @@ object ValidatorsWithSkippedPoW {
   val eip158Validators = EthashValidators(Eip158Config, new EthashTestBlockHeaderValidator(Eip158Config))
   val byzantiumValidators = EthashValidators(ByzantiumConfig, new EthashTestBlockHeaderValidator(ByzantiumConfig))
   val constantinopleValidators = EthashValidators(ConstantinopleConfig, new EthashTestBlockHeaderValidator(ConstantinopleConfig))
+  val constantinopleFixValidators = EthashValidators(ConstantinopleFixConfig, new EthashTestBlockHeaderValidator(ConstantinopleFixConfig))
+  val istanbulValidators = EthashValidators(IstanbulConfig, new EthashTestBlockHeaderValidator(IstanbulConfig))
   val eip158ToByzantiumValidators = EthashValidators(Eip158ToByzantiumAt5Config, new EthashTestBlockHeaderValidator(Eip158ToByzantiumAt5Config))
   val byzantiumToConstantinopleAt5 = EthashValidators(ByzantiumToConstantinopleAt5, new EthashTestBlockHeaderValidator(ByzantiumToConstantinopleAt5))
 }

src/ets/scala/io/iohk/ethereum/ets/blockchain/BlockchainTestConfig.scala

@@ -60,7 +60,7 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
   val ledger =
     new LedgerImpl(
       blockchain,
-      new BlockQueue(blockchain, 10, 10), blockchainConfig, consensus, ScenarioSetup.testContext)
+      new BlockQueue(blockchain, 50, 50), blockchainConfig, consensus, ScenarioSetup.testContext)
 
   def loadGenesis(): Block = {
     val genesisBlock = scenario.genesisRLP match {
@@ -82,20 +82,20 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
 
   val initialWorld: InMemoryWorldStateProxy = InMemoryWorldStateProxy.persistState(getWorldState(scenario.pre))
 
-  val finalWorld: InMemoryWorldStateProxy = InMemoryWorldStateProxy.persistState(getWorldState(scenario.postState))
+  val finalWorld: Option[InMemoryWorldStateProxy] = scenario.postState.map(postState => InMemoryWorldStateProxy.persistState(getWorldState(postState)))
 
   def getBestBlock: Option[Block] = {
     val bestBlockNumber = blockchain.getBestBlockNumber()
     blockchain.getBlockByNumber(bestBlockNumber)
   }
 
-  def getExpectedState: List[(Address, Option[Account])] = {
-    scenario.postState.map((addAcc) => addAcc._1 -> finalWorld.getAccount(addAcc._1)).toList
+  def getExpectedState: Option[List[(Address, Option[Account])]] = {
+    finalWorld.map(w => scenario.postState.get.map(addAcc => addAcc._1 -> w.getAccount(addAcc._1)).toList)
   }
 
-  def getResultState: List[(Address, Option[Account])] = {
+  def getResultState: Option[List[(Address, Option[Account])]] = {
     val bestBlockNumber = blockchain.getBestBlockNumber()
-    scenario.postState.map(addAcc => addAcc._1 -> blockchain.getAccount(addAcc._1, bestBlockNumber)).toList
+    scenario.postState.map(_.map(addAcc => addAcc._1 -> blockchain.getAccount(addAcc._1, bestBlockNumber)).toList)
   }
 
   private def buildBlockchainConfig(network: String, shouldSkipPoW: Boolean): (BlockchainConfig, EthashValidators) = {
@@ -113,7 +113,9 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
     case "Byzantium" => (ByzantiumConfig, Validators.byzantiumValidators)
     case "Constantinople" => (ConstantinopleConfig, Validators.constantinopleValidators)
     case "EIP158ToByzantiumAt5" => (Eip158ToByzantiumAt5Config, Validators.eip158ToByzantiumValidators)
-    case "ByzantiumToConstantinopleAt5" => (ByzantiumToConstantinopleAt5, Validators.byzantiumToConstantinopleAt5)
+    case "ByzantiumToConstantinopleFixAt5" => (ByzantiumToConstantinopleAt5, Validators.byzantiumToConstantinopleAt5)
+    case "ConstantinopleFix" => (ConstantinopleFixConfig, Validators.constantinopleValidators)
+    case "Istanbul" => (IstanbulConfig, Validators.istanbulValidators)
     // Some default config, test will fail or be canceled
     case _ => (FrontierConfig, Validators.frontierValidators)
   }
@@ -129,7 +131,9 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
     case "Byzantium" => (ByzantiumConfig, ValidatorsWithSkippedPoW.byzantiumValidators)
     case "Constantinople" => (ConstantinopleConfig, ValidatorsWithSkippedPoW.constantinopleValidators)
     case "EIP158ToByzantiumAt5" => (Eip158ToByzantiumAt5Config, ValidatorsWithSkippedPoW.eip158ToByzantiumValidators)
-    case "ByzantiumToConstantinopleAt5" => (ByzantiumToConstantinopleAt5, ValidatorsWithSkippedPoW.byzantiumToConstantinopleAt5)
+    case "ByzantiumToConstantinopleFixAt5" => (ByzantiumToConstantinopleAt5, ValidatorsWithSkippedPoW.byzantiumToConstantinopleAt5)
+    case "ConstantinopleFix" => (ConstantinopleFixConfig, ValidatorsWithSkippedPoW.constantinopleFixValidators)
+    case "Istanbul" => (IstanbulConfig, ValidatorsWithSkippedPoW.istanbulValidators)
     // Some default config, test will fail or be canceled
     case _ => (FrontierConfig, ValidatorsWithSkippedPoW.frontierValidators)
   }

src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala

@@ -10,7 +10,8 @@ case class BlockchainScenario(
    genesisRLP: Option[ByteString],
    lastblockhash: ByteString,
    network: String,
-   postState: Map[Address, AccountState],
+   postState: Option[Map[Address, AccountState]],
+   postStateHash: Option[ByteString],
    pre: Map[Address, AccountState],
    sealEngine: Option[String]
 )

src/ets/scala/io/iohk/ethereum/ets/blockchain/scenario.scala

@@ -366,15 +366,17 @@ mantis {
     # depth in ethereum (https://www.reddit.com/r/ethereum/comments/4eplsv/how_many_confirms_is_considered_safe_in_ethereum/)
     branch-resolution-request-size = 12
 
+    # TODO investigate proper value to handle ETC reorgs correctly
     # threshold for storing non-main-chain blocks in queue.
     # if: current_best_block_number - block_number > max-queued-block-number-behind
     # then: the block will not be queued (such already queued blocks will be removed)
-    max-queued-block-number-behind = 10
+    max-queued-block-number-behind = 100
 
+    # TODO investigate proper value to handle ETC reorgs correctly
     # threshold for storing non-main-chain blocks in queue.
     # if: block_number - current_best_block_number > max-queued-block-number-ahead
     # then: the block will not be queued (such already queued blocks will be removed)
-    max-queued-block-number-ahead = 10
+    max-queued-block-number-ahead = 100
 
     # Maximum number of blocks, after which block hash from NewBlockHashes packet is considered ancient
     # and peer sending it is blacklisted

src/main/resources/application.conf

@@ -2,7 +2,7 @@ package io.iohk.ethereum.ledger
 
 import io.iohk.ethereum.domain._
 import io.iohk.ethereum.ledger.Ledger.BlockResult
-import io.iohk.ethereum.utils.{ BlockchainConfig, DaoForkConfig, Logger }
+import io.iohk.ethereum.utils.{BlockchainConfig, DaoForkConfig, Logger}
 import io.iohk.ethereum.vm.EvmConfig
 
 import scala.annotation.tailrec

src/main/scala/io/iohk/ethereum/ledger/BlockExecution.scala

@@ -215,7 +215,7 @@ class BlockImport(
     */
   private def removeBlocksUntil(parent: ByteString, fromNumber: BigInt): List[BlockData] = {
     blockchain.getBlockByNumber(fromNumber) match {
-      case Some(block) if block.header.hash == parent =>
+      case Some(block) if block.header.hash == parent || fromNumber == 0 =>
         Nil
 
       case Some(block) =>

src/main/scala/io/iohk/ethereum/ledger/BlockImport.scala

@@ -151,7 +151,7 @@ class BlockQueueSpec extends FlatSpec with Matchers with MockFactory {
   }
 
   trait TestConfig {
-    val syncConfig = SyncConfig(Config.config)
+    val syncConfig = SyncConfig(Config.config).copy(maxQueuedBlockNumberAhead = 10, maxQueuedBlockNumberBehind = 10)
     val blockchain = mock[BlockchainImpl]
     val blockQueue = BlockQueue(blockchain, syncConfig)