Completed and tested 2.13 migration

Author: Dmitry Voronov

bytes/src/main/scala/io/iohk/ethereum/utils/ByteStringUtils.scala

@@ -2,6 +2,9 @@ package io.iohk.ethereum.utils
 
 import akka.util.ByteString
 
+import scala.collection.mutable
+import scala.math.Ordering.Implicits._
+
 object ByteStringUtils {
   def hash2string(hash: ByteString): String =
     Hex.toHexString(hash.toArray[Byte])
@@ -48,19 +51,19 @@ object ByteStringUtils {
     def asByteArray: Array[Byte] = Array(b)
   }
 
-  def concatByteStrings(bse: ByteStringElement*): ByteString = {
-    concatByteStrings(bse.toIterable)
+  implicit val byteStringOrdering: Ordering[ByteString] = {
+    Ordering.by[ByteString, Seq[Byte]](_.toSeq)
+  }
+
+  def concatByteStrings(head: ByteStringElement, tail: ByteStringElement*): ByteString = {
+    val it = Iterator.single(head) ++ tail.iterator
+    concatByteStrings(it)
   }
 
-  def concatByteStrings(bse: Iterable[ByteStringElement]): ByteString = {
-    val totalLength = bse.map(_.len).sum
-    val result = new Array[Byte](totalLength)
-    bse.foldLeft(0)( (i, el) => {
-      val arr = el.asByteArray
-      System.arraycopy(arr, 0, result, i, el.len)
-      i + el.len
-    })
-    ByteString.fromArray(result)
+  def concatByteStrings(elements: Iterator[ByteStringElement]): ByteString = {
+    val builder = new mutable.ArrayBuilder.ofByte
+    elements.foreach(el => builder.addAll(el.asByteArray))
+    ByteString(builder.result())
   }
 
 }

bytes/src/test/scala/io/iohk/ethereum/utils/ByteStringUtilsTest.scala

@@ -0,0 +1,21 @@
+package io.iohk.ethereum.utils
+
+import akka.util.ByteString
+import org.scalatest.wordspec.AnyWordSpec
+
+class ByteStringUtilsTest extends AnyWordSpec {
+
+  "ByteStringUtilsTest" should {
+
+    "concatByteStrings for simple bytestrings" in {
+      val bs1 = ByteString("000")
+      val bs2 = ByteString("111")
+      val summarized: ByteString = bs1 ++ bs2
+
+      val concatenated: ByteString = ByteStringUtils.concatByteStrings(bs1, bs2)
+      println(s"Bs1: ${bs1} Bs2: ${bs2} Sum: ${summarized} Concat: ${concatenated}")
+      assert(summarized == concatenated)
+    }
+
+  }
+}

project/Dependencies.scala

@@ -60,8 +60,8 @@ object Dependencies {
   val testing: Seq[ModuleID] = Seq(
     "org.scalatest" %% "scalatest" % "3.2.2" % "it,test",
     "org.scalamock" %% "scalamock" % "5.0.0" % "test",
-    "org.scalatestplus" %% "scalacheck-1-14" % "3.2.2.0" % "test",
-    "org.scalacheck" %% "scalacheck" % "1.14.3" % "it,test",
+    "org.scalatestplus" %% "scalacheck-1-15" % "3.2.3.0" % "test",
+    "org.scalacheck" %% "scalacheck" % "1.15.1" % "it,test",
     "com.softwaremill.diffx" %% "diffx-core" % "0.3.30" % "test",
     "com.softwaremill.diffx" %% "diffx-scalatest" % "0.3.30" % "test"
   )

src/benchmark/scala/io/iohk/ethereum/rlp/RLPSpeedSuite.scala

@@ -57,7 +57,7 @@ class RLPSpeedSuite
     val ITERATIONS: Int = 10000000
     log.info("Starting " + ITERATIONS + " decoding iterations...")
     val start1: Long = System.currentTimeMillis
-    (1 to ITERATIONS).foreach { _ => RLP.rawDecode(payload); Unit }
+    (1 to ITERATIONS).foreach { _ => RLP.rawDecode(payload); () }
     val end1: Long = System.currentTimeMillis
     log.info("Result decode()\t: " + (end1 - start1) + "ms")
   }

src/it/scala/io/iohk/ethereum/db/DataSourceIntegrationTestBehavior.scala

@@ -9,6 +9,7 @@ import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceUpdate}
 import io.iohk.ethereum.db.dataSource.DataSource.{Key, Namespace, Value}
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
+import io.iohk.ethereum.utils.ByteStringUtils._
 
 trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with ObjectGenerators {
 
@@ -91,7 +92,7 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob
           val db = createDataSource(path)
           db.update(prepareUpdate(toUpsert = keyList.zip(keyList)))
 
-          val keyListWithExtraByte = keyList.map(1.toByte +: _)
+          val keyListWithExtraByte = keyList.map(key => concatByteStrings(1.toByte, key))
           updateInSeparateCalls(db, keyList.zip(keyListWithExtraByte))
 
           keyList.zip(keyListWithExtraByte).foreach { case (key, value) =>

src/it/scala/io/iohk/ethereum/db/RockDbIteratorSpec.scala

@@ -45,7 +45,7 @@ class RockDbIteratorSpec extends FlatSpecBase with ResourceFixtures with Matcher
       _ <- writeNValuesToDb(largeNum, db, Namespaces.NodeNamespace)
       fib <- db
         .iterate(Namespaces.NodeNamespace)
-        .map(_.right.get)
+        .map(_.toOption.get)
         .consumeWith(Consumer.foreachEval[Task, (Array[Byte], Array[Byte])] { _ =>
           for {
             cur <- counter.updateAndGet(i => i + 1)
@@ -71,7 +71,7 @@ class RockDbIteratorSpec extends FlatSpecBase with ResourceFixtures with Matcher
       _ <- writeNValuesToDb(largeNum, db, Namespaces.NodeNamespace)
       _ <- db
         .iterate(Namespaces.NodeNamespace)
-        .map(_.right.get)
+        .map(_.toOption.get)
         .consumeWith(Consumer.foreachEval[Task, (Array[Byte], Array[Byte])] { _ =>
           counter.update(current => current + 1)
         })
@@ -92,8 +92,8 @@ class RockDbIteratorSpec extends FlatSpecBase with ResourceFixtures with Matcher
       _ <- Task(codeStorage.update(Seq(), codeKeyValues).commit())
       _ <- Task(nodeStorage.update(Seq(), nodeKeyValues))
       result <- Task.parZip2(
-        codeStorage.storageContent.map(_.right.get).map(_._1).toListL,
-        nodeStorage.storageContent.map(_.right.get).map(_._1).toListL
+        codeStorage.storageContent.map(_.toOption.get).map(_._1).toListL,
+        nodeStorage.storageContent.map(_.toOption.get).map(_._1).toListL
       )
       (codeResult, nodeResult) = result
     } yield {
@@ -112,7 +112,7 @@ class RockDbIteratorSpec extends FlatSpecBase with ResourceFixtures with Matcher
           )
         )
       )
-      elems <- db.iterate(Namespaces.NodeNamespace).map(_.right.get).toListL
+      elems <- db.iterate(Namespaces.NodeNamespace).map(_.toOption.get).toListL
     } yield {
       val deserialized = elems.map { case (bytes, bytes1) => (ByteString(bytes), ByteString(bytes1)) }
       assert(elems.size == keyValues.size)

src/main/scala/io/iohk/ethereum/db/dataSource/DataSourceBatchUpdate.scala

@@ -1,6 +1,6 @@
 package io.iohk.ethereum.db.dataSource
 
-import scala.collection.compat.immutable.ArraySeq
+import scala.collection.immutable.ArraySeq
 
 case class DataSourceBatchUpdate(dataSource: DataSource, updates: Array[DataUpdate] = Array.empty) {
 

src/main/scala/io/iohk/ethereum/db/dataSource/RocksDbDataSource.scala

@@ -11,7 +11,7 @@ import monix.eval.Task
 import monix.reactive.Observable
 import org.rocksdb._
 
-import scala.collection.compat.immutable.ArraySeq
+import scala.collection.immutable.ArraySeq
 import scala.collection.mutable
 import scala.util.control.NonFatal
 

src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala

@@ -5,7 +5,7 @@ import java.math.BigInteger
 import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceBatchUpdate}
 import io.iohk.ethereum.db.storage.AppStateStorage._
 
-import scala.collection.compat.immutable.ArraySeq
+import scala.collection.immutable.ArraySeq
 
 /**
   * This class is used to store app state variables

src/main/scala/io/iohk/ethereum/db/storage/FastSyncStateStorage.scala

@@ -9,7 +9,7 @@ import io.iohk.ethereum.blockchain.sync.fast.FastSync._
 import io.iohk.ethereum.db.dataSource.DataSource
 import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes
 
-import scala.collection.compat.immutable.ArraySeq
+import scala.collection.immutable.ArraySeq
 
 object FastSyncStateStorage {
 

src/main/scala/io/iohk/ethereum/jsonrpc/server/controllers/JsonRpcBaseController.scala

@@ -14,7 +14,7 @@ import monix.eval.Task
 import org.json4s.JsonDSL._
 import org.json4s.{DefaultFormats, native}
 
-import scala.collection.compat.immutable.ArraySeq
+import scala.collection.immutable.ArraySeq
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.FiniteDuration
 

src/main/scala/io/iohk/ethereum/network/rlpx/FrameCodec.scala

@@ -172,7 +172,7 @@ class FrameCodec(private val secrets: Secrets) {
       elements.append(bytes)
       i += bytes.length
     }
-    concatByteStrings(elements)
+    concatByteStrings(elements.iterator)
   }
 
   private def processFramePadding(totalSize: Int): ByteString = {

src/test/scala/io/iohk/ethereum/ObjectGenerators.scala

@@ -19,8 +19,6 @@ import org.scalacheck.{Arbitrary, Gen, Shrink}
 // scalastyle:off number.of.methods
 trait ObjectGenerators {
 
-  def noShrink[T]: Shrink[T] = Shrink[T](_ => Stream.empty)
-
   def byteGen: Gen[Byte] = Gen.choose(Byte.MinValue, Byte.MaxValue)
 
   def shortGen: Gen[Short] = Gen.choose(Short.MinValue, Short.MaxValue)

src/test/scala/io/iohk/ethereum/blockchain/sync/SyncControllerSpec.scala

@@ -536,7 +536,7 @@ class SyncControllerSpec extends AnyFlatSpec with Matchers with BeforeAndAfter w
               sender ! MessageFromPeer(BlockHeaders(Seq(pivotHeader)), peer)
             } else {
               if (!onlyPivot) {
-                val start = msg.underlyingMsg.block.left.get
+                val start = msg.underlyingMsg.block.swap.toOption.get
                 val stop = start + msg.underlyingMsg.maxHeaders
                 val headers = (start until stop).flatMap(i => blockchainData.headers.get(i))
                 sender ! MessageFromPeer(BlockHeaders(headers), peer)

src/test/scala/io/iohk/ethereum/blockchain/sync/regular/RegularSyncSpec.scala

@@ -358,13 +358,11 @@ class RegularSyncSpec
       })
 
       "save fetched node" in sync(new Fixture(testSystem) {
-        implicit val ec: Scheduler = Scheduler(system.dispatcher)
         override lazy val blockchain: BlockchainImpl = stub[BlockchainImpl]
         override lazy val ledger: TestLedgerImpl = stub[TestLedgerImpl]
         val failingBlock: Block = testBlocksChunked.head.head
         peersClient.setAutoPilot(new PeersClientAutoPilot)
-
-        (ledger.resolveBranch _).when(*).returns(NewBetterBranch(Nil))
+        (ledger.resolveBranch _).when(*).returns(NewBetterBranch(Nil)).repeat(10)
         (ledger
           .importBlock(_: Block)(_: Scheduler))
           .when(*, *)

src/test/scala/io/iohk/ethereum/cli/CliCommandsSpec.scala

@@ -82,7 +82,7 @@ class CliCommandsSpec extends AnyFlatSpec with Matchers with EitherValues {
   it should "generate one key pair when passed no args" in {
     val result = api.parse(Seq(generateKeyPairsCommand))
     result shouldBe a[Right[_, _]]
-    val stringSplit = result.right.get.split("\\n\\n")
+    val stringSplit = result.toOption.get.split("\\n\\n")
     stringSplit.length shouldEqual 1
   }
 
@@ -91,7 +91,7 @@ class CliCommandsSpec extends AnyFlatSpec with Matchers with EitherValues {
     val numOfKeysAsInt = numOfKeys.toInt
     val result = api.parse(Seq(generateKeyPairsCommand, numOfKeys))
     result shouldBe a[Right[_, _]]
-    val stringSplit = result.right.get.split("\\n\\n")
+    val stringSplit = result.toOption.get.split("\\n\\n")
     stringSplit.length shouldEqual numOfKeysAsInt
   }
 

src/test/scala/io/iohk/ethereum/consensus/validators/BlockWithCheckpointHeaderValidatorSpec.scala

@@ -19,6 +19,7 @@ import org.scalatest.Assertion
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
+import ByteStringUtils.byteStringOrdering
 
 class BlockWithCheckpointHeaderValidatorSpec
     extends AnyFlatSpec
@@ -224,9 +225,7 @@ class BlockWithCheckpointHeaderValidatorSpec
     import Ordering.Implicits._
     val actualSigners = invalidCheckpoint.signatures.flatMap(_.publicKey(validBlockParent.hash)).sortBy(_.toSeq)
     val duplicatedSigner = ByteString(crypto.pubKeyFromKeyPair(keys.head))
-    val expectedSigners =
-      (keys.map(kp => ByteString(crypto.pubKeyFromKeyPair(kp))) :+ duplicatedSigner)
-        .sortBy(_.toSeq)
+    val expectedSigners = (keys.map(kp => ByteString(crypto.pubKeyFromKeyPair(kp))) :+ duplicatedSigner).sorted
     actualSigners shouldEqual expectedSigners
 
     val headerWithInvalidCheckpoint = checkpointBlockGenerator

src/test/scala/io/iohk/ethereum/domain/ArbitraryIntegerMptSpec.scala

@@ -11,8 +11,6 @@ import org.scalatest.matchers.should.Matchers
 
 class ArbitraryIntegerMptSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyChecks {
 
-  implicit val bigIntListNoShrink = noShrink[List[BigInt]]
-
   def keyGen: Gen[BigInt] = byteArrayOfNItemsGen(128).map(BigInt.apply)
   def valueGen: Gen[BigInt] = byteArrayOfNItemsGen(128).map(BigInt.apply)
 

src/test/scala/io/iohk/ethereum/extvm/MessageHandlerSpec.scala

@@ -5,15 +5,19 @@ import akka.stream.OverflowStrategy
 import akka.stream.scaladsl.{Keep, Sink, SinkQueueWithCancel, Source, SourceQueueWithComplete}
 import akka.testkit.TestProbe
 import akka.util.ByteString
-import scalapb.GeneratedMessage
+import scalapb.{GeneratedMessage, GeneratedMessageCompanion}
 import io.iohk.ethereum.vm.Generators
 import java.math.BigInteger
+
+import com.google.protobuf.CodedOutputStream
 import org.bouncycastle.util.BigIntegers
 import org.scalamock.scalatest.MockFactory
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
+
 import scala.concurrent.ExecutionContext.Implicits.global
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
+import scalapb.descriptors.{FieldDescriptor, PValue}
 
 class MessageHandlerSpec extends AnyFlatSpec with Matchers with MockFactory with ScalaCheckPropertyChecks {
 
@@ -33,9 +37,16 @@ class MessageHandlerSpec extends AnyFlatSpec with Matchers with MockFactory with
       val (out, fut) = Source.queue[ByteString](1024, OverflowStrategy.dropTail).toMat(Sink.seq)(Keep.both).run()
       fut.pipeTo(probe.ref)
 
-      val gm = mock[GeneratedMessage]
-      (() => gm.toByteArray).expects().returning(bytes.toArray[Byte])
-
+      // mock of final fields is no longer possible
+      // had to create a GeneratedMessage stub
+      val gm = new GeneratedMessage {
+        override def writeTo(output: CodedOutputStream): Unit = output.writeRawBytes(bytes)
+        override def getFieldByNumber(fieldNumber: Int): Any = ???
+        override def getField(field: FieldDescriptor): PValue = ???
+        override def companion: GeneratedMessageCompanion[_] = ???
+        override def serializedSize: Int = bytes.size
+        override def toProtoString: String = ???
+      }
       val messageHandler = new MessageHandler(in, out)
       messageHandler.sendMessage(gm)
       messageHandler.close()