Merge remote-tracking branch 'origin/develop' into etcm-213/relad-bloom-after-restart

Author: KonradStaniec

build.sbt

@@ -146,3 +146,13 @@ addCommandAlias(
     |;benchmark:compile
     |""".stripMargin
 )
+
+// prepare PR
+addCommandAlias(
+  "pp",
+  """;compile-all
+    |;test
+    |;scalastyle
+    |;test:scalastyle
+    |""".stripMargin
+)

insomnia_workspace.json

@@ -139,43 +139,6 @@
       "metaSortKey": -1600249374160,
       "_type": "request_group"
     },
-    {
-      "_id": "req_6197fefa1e1448a89f30712ec12295f8",
-      "parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
-      "modified": 1600249571669,
-      "created": 1600249491560,
-      "url": "{{ node_url }}",
-      "name": "qa_getPendingTransactions",
-      "description": "",
-      "method": "POST",
-      "body": {
-        "mimeType": "application/json",
-        "text": "{\n\t\"jsonrpc\": \"2.0\",\n  \"method\": \"qa_getPendingTransactions\", \n  \"params\": [],\n  \"id\": 1\n}"
-      },
-      "parameters": [],
-      "headers": [
-        {
-          "id": "pair_9f4d6a9dde554cd384487e04fa3b21aa",
-          "name": "Content-Type",
-          "value": "application/json"
-        },
-        {
-          "id": "pair_088edc31f5e04f20a16b465a673871bb",
-          "name": "Cache-Control",
-          "value": "no-cache"
-        }
-      ],
-      "authentication": {},
-      "metaSortKey": -1552939150156.832,
-      "isPrivate": false,
-      "settingStoreCookies": true,
-      "settingSendCookies": true,
-      "settingDisableRenderRequestBody": false,
-      "settingEncodeUrl": true,
-      "settingRebuildPath": true,
-      "settingFollowRedirects": "global",
-      "_type": "request"
-    },
     {
       "_id": "req_dd6e5c718f974407bb79fe3c953b7106",
       "parentId": "fld_72829b866f0441e184e0d1a2030f8220",
@@ -961,6 +924,43 @@
       "settingFollowRedirects": "global",
       "_type": "request"
     },
+    {
+      "_id": "req_4c1135a4a69644fe9850292131197b47",
+      "parentId": "fld_a06eb77e183c4727800eb7dc43ceabe1",
+      "modified": 1602234543563,
+      "created": 1602234438767,
+      "url": "{{ node_url }}",
+      "name": "eth_pendingTransactions",
+      "description": "Returns the transactions that are pending in the transaction pool and have a from address that is one of the accounts this node manages",
+      "method": "POST",
+      "body": {
+        "mimeType": "application/json",
+        "text": "{\n\t\"jsonrpc\": \"2.0\",\n  \"method\": \"eth_pendingTransactions\", \n\t\"params\": [],\n  \"id\": 1\n}"
+      },
+      "parameters": [],
+      "headers": [
+        {
+          "id": "pair_9f4d6a9dde554cd384487e04fa3b21aa",
+          "name": "Content-Type",
+          "value": "application/json"
+        },
+        {
+          "id": "pair_088edc31f5e04f20a16b465a673871bb",
+          "name": "Cache-Control",
+          "value": "no-cache"
+        }
+      ],
+      "authentication": {},
+      "metaSortKey": -1552732410719.375,
+      "isPrivate": false,
+      "settingStoreCookies": true,
+      "settingSendCookies": true,
+      "settingDisableRenderRequestBody": false,
+      "settingEncodeUrl": true,
+      "settingRebuildPath": true,
+      "settingFollowRedirects": "global",
+      "_type": "request"
+    },
     {
       "_id": "req_579a127b434b4ac7aa51ee6c93f04630",
       "parentId": "fld_a06eb77e183c4727800eb7dc43ceabe1",

src/main/resources/application.conf

@@ -261,17 +261,17 @@ mantis {
   blockchains {
     network = "etc"
 
-    etc { include "chains/etc-chain.conf"}
+    etc {include "chains/etc-chain.conf"}
 
-    eth { include "chains/eth-chain.conf"}
+    eth {include "chains/eth-chain.conf"}
 
-    mordor { include "chains/mordor-chain.conf"}
+    mordor {include "chains/mordor-chain.conf"}
 
-    ropsten { include "chains/ropsten-chain.conf"}
+    ropsten {include "chains/ropsten-chain.conf"}
 
-    test { include "chains/test-chain.conf"}
+    test {include "chains/test-chain.conf"}
 
-    testnet-internal { include "chains/testnet-internal-chain.conf"}
+    testnet-internal {include "chains/testnet-internal-chain.conf"}
   }
 
   sync {
@@ -397,17 +397,16 @@ mantis {
     # Current Size of ETC state trie is aroud 150M Nodes, so 200M is set to have some reserve
     # If the number of elements inserted into bloom filter would be significally higher that expected, then number
     # of false positives would rise which would degrade performance of state sync
-    state-sync-bloomFilter-size = 200000000
-
+    state-sync-bloom-filter-size = 200000000
 
     # Max number of mpt nodes held in memory in state sync, before saving them into database
     # 100k is around 60mb (each key-value pair has around 600bytes)
-    state-sync-persistBatch-size = 100000
+    state-sync-persist-batch-size = 100000
 
     # If new pivot block received from network will be less than fast sync current pivot block, the re-try to chose new
     # pivot will be scheduler after this time. Avarage block time in etc/eth is around 15s so after this time, most of
     # network peers should have new best block
-    pivot-block-reSchedule-interval =  15.seconds
+    pivot-block-reschedule-interval = 15.seconds
 
     # If for most network peers, the following condition will be true:
     # (peer.bestKnownBlock - pivot-block-offset) - node.curentPivotBlock > max-pivot-age

src/main/scala/io/iohk/ethereum/blockchain/sync/FastSync.scala

@@ -72,7 +72,7 @@ class FastSync(
   def startWithState(syncState: SyncState): Unit = {
     log.info(s"Starting with existing state and asking for new pivot block")
     val syncingHandler = new SyncingHandler(syncState)
-    syncingHandler.askForPivotBlockUpdate(NodeRestart)
+    syncingHandler.askForPivotBlockUpdate(SyncRestart)
   }
 
   def startFromScratch(): Unit = {
@@ -205,39 +205,48 @@ class FastSync(
       scheduler.scheduleOnce(syncConfig.pivotBlockReScheduleInterval, self, UpdatePivotBlock(updateReason))
     }
 
+    private def stalePivotAfterRestart(
+        newPivot: BlockHeader,
+        currentState: SyncState,
+        updateReason: PivotBlockUpdateReason
+    ): Boolean = {
+      newPivot.number == currentState.pivotBlock.number && updateReason.isSyncRestart
+    }
+
+    private def newPivotIsGoodEnough(
+        newPivot: BlockHeader,
+        currentState: SyncState,
+        updateReason: PivotBlockUpdateReason
+    ): Boolean = {
+      newPivot.number >= currentState.pivotBlock.number && !stalePivotAfterRestart(newPivot, currentState, updateReason)
+    }
+
     def waitingForPivotBlockUpdate(updateReason: PivotBlockUpdateReason): Receive = handleCommonMessages orElse {
-      case PivotBlockSelector.Result(pivotBlockHeader) =>
+      case PivotBlockSelector.Result(pivotBlockHeader)
+          if newPivotIsGoodEnough(pivotBlockHeader, syncState, updateReason) =>
         log.info(s"New pivot block with number ${pivotBlockHeader.number} received")
-        if (pivotBlockHeader.number >= syncState.pivotBlock.number) {
-          if (pivotBlockHeader.number == syncState.pivotBlock.number && updateReason.nodeRestart) {
-            // it can happen after quick node restart than pivot block has not changed in the network. To keep whole
-            // fast sync machinery running as expected we need to make sure that we will receive better pivot than current
-            log.info("Received stale pivot after restart, asking for new pivot")
-            reScheduleAskForNewPivot(updateReason)
-          } else {
-            updatePivotSyncState(updateReason, pivotBlockHeader)
-            syncState = syncState.copy(updatingPivotBlock = false)
-            context become this.receive
-            processSyncing()
-          }
-        } else {
-          log.info("Received target block is older than old one, re-scheduling asking for new one")
-          reScheduleAskForNewPivot(updateReason)
-        }
+        updatePivotSyncState(updateReason, pivotBlockHeader)
+        context become this.receive
+        processSyncing()
+
+      case PivotBlockSelector.Result(pivotBlockHeader)
+          if !newPivotIsGoodEnough(pivotBlockHeader, syncState, updateReason) =>
+        log.info("Received pivot block is older than old one, re-scheduling asking for new one")
+        reScheduleAskForNewPivot(updateReason)
 
       case PersistSyncState => persistSyncState()
 
       case UpdatePivotBlock(state) => updatePivotBlock(state)
     }
 
-    private def updatePivotBlock(state: PivotBlockUpdateReason): Unit = {
+    private def updatePivotBlock(updateReason: PivotBlockUpdateReason): Unit = {
       if (syncState.pivotBlockUpdateFailures <= syncConfig.maximumTargetUpdateFailures) {
         if (assignedHandlers.nonEmpty || syncState.blockChainWorkQueued) {
           log.info(s"Still waiting for some responses, rescheduling pivot block update")
-          scheduler.scheduleOnce(1.second, self, UpdatePivotBlock(state))
+          scheduler.scheduleOnce(1.second, self, UpdatePivotBlock(updateReason))
           processSyncing()
         } else {
-          askForPivotBlockUpdate(state)
+          askForPivotBlockUpdate(updateReason)
         }
       } else {
         log.warning(s"Sync failure! Number of pivot block update failures reached maximum.")
@@ -253,8 +262,9 @@ class FastSync(
             // Empty root has means that there were no transactions in blockchain, and Mpt trie is empty
             // Asking for this root would result only with empty transactions
             if (syncState.pivotBlock.stateRoot == ByteString(MerklePatriciaTrie.EmptyRootHash)) {
-              syncState = syncState.copy(stateSyncFinished = true)
+              syncState = syncState.copy(stateSyncFinished = true, updatingPivotBlock = false)
             } else {
+              syncState = syncState.copy(updatingPivotBlock = false)
               stateSyncRestartRequested = false
               syncStateScheduler ! StartSyncingTo(pivotBlockHeader.stateRoot, pivotBlockHeader.number)
             }
@@ -276,7 +286,7 @@ class FastSync(
           syncState =
             syncState.updatePivotBlock(pivotBlockHeader, syncConfig.fastSyncBlockValidationX, updateFailures = true)
 
-        case NodeRestart =>
+        case SyncRestart =>
           // in case of node restart we are sure that new pivotBlockHeader > current pivotBlockHeader
           syncState = syncState.updatePivotBlock(
             pivotBlockHeader,
@@ -541,7 +551,7 @@ class FastSync(
       (info.maxBlockNumber - syncConfig.pivotBlockOffset) - state.pivotBlock.number >= syncConfig.maxPivotBlockAge
     }
 
-    private def getPeerWithTooFreshNewBlock(
+    private def getPeersWithFreshEnoughPivot(
         peers: NonEmptyList[(Peer, PeerInfo)],
         state: SyncState,
         syncConfig: SyncConfig
@@ -569,7 +579,7 @@ class FastSync(
           (peerWithBestBlockInNetwork._2.maxBlockNumber - syncConfig.pivotBlockOffset) - syncState.pivotBlock.number
 
         val peersWithTooFreshPossiblePivotBlock =
-          getPeerWithTooFreshNewBlock(NonEmptyList.fromListUnsafe(currentPeers), syncState, syncConfig)
+          getPeersWithFreshEnoughPivot(NonEmptyList.fromListUnsafe(currentPeers), syncState, syncConfig)
 
         if (peersWithTooFreshPossiblePivotBlock.isEmpty) {
           log.info(
@@ -830,7 +840,8 @@ object FastSync {
       copy(
         pivotBlock = newPivot,
         safeDownloadTarget = newPivot.number + numberOfSafeBlocks,
-        pivotBlockUpdateFailures = if (updateFailures) pivotBlockUpdateFailures + 1 else pivotBlockUpdateFailures
+        pivotBlockUpdateFailures = if (updateFailures) pivotBlockUpdateFailures + 1 else pivotBlockUpdateFailures,
+        updatingPivotBlock = false
       )
 
     def isBlockchainWorkFinished: Boolean = {
@@ -861,14 +872,14 @@ object FastSync {
   case object ImportedPivotBlock extends HeaderProcessingResult
 
   sealed abstract class PivotBlockUpdateReason {
-    def nodeRestart: Boolean = this match {
+    def isSyncRestart: Boolean = this match {
       case ImportedLastBlock => false
       case LastBlockValidationFailed => false
-      case NodeRestart => true
+      case SyncRestart => true
     }
   }
 
   case object ImportedLastBlock extends PivotBlockUpdateReason
   case object LastBlockValidationFailed extends PivotBlockUpdateReason
-  case object NodeRestart extends PivotBlockUpdateReason
+  case object SyncRestart extends PivotBlockUpdateReason
 }

src/main/scala/io/iohk/ethereum/blockchain/sync/SyncController.scala

@@ -19,8 +19,8 @@ class SyncController(
     ommersPool: ActorRef,
     etcPeerManager: ActorRef,
     syncConfig: SyncConfig,
-    externalSchedulerOpt: Option[Scheduler] = None)
-    extends Actor
+    externalSchedulerOpt: Option[Scheduler] = None
+) extends Actor
     with ActorLogging {
 
   import SyncController._
@@ -29,8 +29,8 @@ class SyncController(
 
   override def receive: Receive = idle
 
-  def idle: Receive = {
-    case Start => start()
+  def idle: Receive = { case Start =>
+    start()
   }
 
   def runningFastSync(fastSync: ActorRef): Receive = {
@@ -41,8 +41,8 @@ class SyncController(
     case other => fastSync.forward(other)
   }
 
-  def runningRegularSync(regularSync: ActorRef): Receive = {
-    case other => regularSync.forward(other)
+  def runningRegularSync(regularSync: ActorRef): Receive = { case other =>
+    regularSync.forward(other)
   }
 
   def start(): Unit = {
@@ -54,15 +54,17 @@ class SyncController(
         startFastSync()
       case (true, true) =>
         log.warning(
-          s"do-fast-sync is set to $doFastSync but fast sync cannot start because it has already been completed")
+          s"do-fast-sync is set to $doFastSync but fast sync cannot start because it has already been completed"
+        )
         startRegularSync()
       case (true, false) =>
         startRegularSync()
       case (false, false) =>
         //Check whether fast sync was started before
         if (fastSyncStateStorage.getSyncState().isDefined) {
           log.warning(
-            s"do-fast-sync is set to $doFastSync but regular sync cannot start because fast sync hasn't completed")
+            s"do-fast-sync is set to $doFastSync but regular sync cannot start because fast sync hasn't completed"
+          )
           startFastSync()
         } else
           startRegularSync()
@@ -79,14 +81,17 @@ class SyncController(
         peerEventBus,
         etcPeerManager,
         syncConfig,
-        scheduler),
-      "fast-sync")
+        scheduler
+      ),
+      "fast-sync"
+    )
     fastSync ! FastSync.Start
     context become runningFastSync(fastSync)
   }
 
   def startRegularSync(): Unit = {
-    val peersClient = context.actorOf(PeersClient.props(etcPeerManager, peerEventBus, syncConfig, scheduler), "peers-client")
+    val peersClient =
+      context.actorOf(PeersClient.props(etcPeerManager, peerEventBus, syncConfig, scheduler), "peers-client")
     val regularSync = context.actorOf(
       RegularSync.props(
         peersClient,
@@ -97,7 +102,8 @@ class SyncController(
         syncConfig,
         ommersPool,
         pendingTransactionsManager,
-        scheduler),
+        scheduler
+      ),
       "regular-sync"
     )
     regularSync ! RegularSync.Start
@@ -118,7 +124,8 @@ object SyncController {
       pendingTransactionsManager: ActorRef,
       ommersPool: ActorRef,
       etcPeerManager: ActorRef,
-      syncConfig: SyncConfig): Props =
+      syncConfig: SyncConfig
+  ): Props =
     Props(
       new SyncController(
         appStateStorage,
@@ -130,7 +137,9 @@ object SyncController {
         pendingTransactionsManager,
         ommersPool,
         etcPeerManager,
-        syncConfig))
+        syncConfig
+      )
+    )
 
   case object Start
 }