ETCM-167: Scalanet Discovery part 3

build.sbt

@@ -12,6 +12,8 @@ val commonSettings = Seq(
   name := "mantis",
   version := "3.0",
   scalaVersion := "2.12.12",
+  // Scalanet snapshots are published to Sonatype after each build.
+  resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
   testOptions in Test += Tests
     .Argument(TestFrameworks.ScalaTest, "-l", "EthashMinerSpec") // miner tests disabled by default
 )
@@ -35,6 +37,7 @@ val dep = {
     Dependencies.testing,
     Dependencies.cats,
     Dependencies.monix,
+    Dependencies.network,
     Dependencies.twitterUtilCollection,
     Dependencies.crypto,
     Dependencies.scopt,

project/Dependencies.scala

@@ -70,6 +70,14 @@ object Dependencies {
     "io.monix" %% "monix" % "3.2.2"
   )
 
+  val network: Seq[ModuleID] = {
+    val scalanetVersion = "0.4-SNAPSHOT"
+    Seq(
+      "io.iohk" %% "scalanet" % scalanetVersion,
+      "io.iohk" %% "scalanet-discovery" % scalanetVersion
+    )
+  }
+
   val logging = Seq(
     "ch.qos.logback" % "logback-classic" % "1.2.3",
     "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2",

src/evmTest/scala/io/iohk/ethereum/vm/PrecompiledContractsSpecEvm.scala

@@ -14,7 +14,7 @@ class PrecompiledContractsSpecEvm extends AnyFunSuite with Matchers with SecureR
 
   test("Precompiled Contracts") {
     val keyPair = generateKeyPair(secureRandom)
-    val bytes: Array[Byte] = ByteString("aabbccdd").toArray[Byte]
+    val bytes: Array[Byte] = crypto.kec256(ByteString("aabbccdd").toArray[Byte])
     val signature = ECDSASignature.sign(bytes, keyPair)
     val pubKey = keyPair.getPublic.asInstanceOf[ECPublicKeyParameters].getQ.getEncoded(false)
     val address = crypto.kec256(pubKey.tail).slice(FirstByteOfAddress, LastByteOfAddress)

src/main/scala/io/iohk/ethereum/crypto/ECDSASignature.scala

@@ -15,8 +15,11 @@ object ECDSASignature {
   val RLength = 32
   val VLength = 1
   val EncodedLength: Int = RLength + SLength + VLength
+
   //byte value that indicates that bytes representing ECC point are in uncompressed format, and should be decoded properly
-  val uncompressedIndicator: Byte = 0x04
+  val UncompressedIndicator: Byte = 0x04
+  val CompressedEvenIndicator: Byte = 0x02
+  val CompressedOddIndicator: Byte = 0x03
 
   //only naming convention
   val negativePointSign: Byte = 27
@@ -37,17 +40,22 @@ object ECDSASignature {
       None
   }
 
-  def sign(message: ByteString, prvKey: ByteString): ECDSASignature =
-    sign(message.toArray, keyPairFromPrvKey(prvKey.toArray), None)
+  def sign(messageHash: ByteString, prvKey: ByteString): ECDSASignature =
+    sign(messageHash.toArray, keyPairFromPrvKey(prvKey.toArray), None)
 
-  def sign(message: Array[Byte], keyPair: AsymmetricCipherKeyPair, chainId: Option[Byte] = None): ECDSASignature = {
+  /** Sign a messageHash, expected to be a Keccak256 hash of the original data. */
+  def sign(messageHash: Array[Byte], keyPair: AsymmetricCipherKeyPair, chainId: Option[Byte] = None): ECDSASignature = {
+    require(
+      messageHash.size == 32,
+      s"The message should be a hash, expected to be 32 bytes; got ${messageHash.size} bytes."
+    )
     val signer = new ECDSASigner(new HMacDSAKCalculator(new SHA256Digest))
     signer.init(true, keyPair.getPrivate)
-    val components = signer.generateSignature(message)
+    val components = signer.generateSignature(messageHash)
     val r = components(0)
     val s = ECDSASignature.canonicalise(components(1))
     val v = ECDSASignature
-      .calculateV(r, s, keyPair, message)
+      .calculateV(r, s, keyPair, messageHash)
       .getOrElse(throw new RuntimeException("Failed to calculate signature rec id"))
 
     val pointSign = chainId match {
@@ -83,11 +91,11 @@ object ECDSASignature {
     else s
   }
 
-  private def calculateV(r: BigInt, s: BigInt, key: AsymmetricCipherKeyPair, message: Array[Byte]): Option[Byte] = {
+  private def calculateV(r: BigInt, s: BigInt, key: AsymmetricCipherKeyPair, messageHash: Array[Byte]): Option[Byte] = {
     //byte 0 of encoded ECC point indicates that it is uncompressed point, it is part of bouncycastle encoding
     val pubKey = key.getPublic.asInstanceOf[ECPublicKeyParameters].getQ.getEncoded(false).tail
     val recIdOpt = Seq(positivePointSign, negativePointSign).find { i =>
-      recoverPubBytes(r, s, i, None, message).exists(java.util.Arrays.equals(_, pubKey))
+      recoverPubBytes(r, s, i, None, messageHash).exists(java.util.Arrays.equals(_, pubKey))
     }
     recIdOpt
   }
@@ -97,7 +105,7 @@ object ECDSASignature {
       s: BigInt,
       recId: Byte,
       chainId: Option[Byte],
-      message: Array[Byte]
+      messageHash: Array[Byte]
   ): Option[Array[Byte]] = {
     val order = curve.getCurve.getOrder
     //ignore case when x = r + order because it is negligibly improbable
@@ -110,7 +118,7 @@ object ECDSASignature {
       if (xCoordinate.compareTo(prime) < 0) {
         val R = constructPoint(xCoordinate, recovery)
         if (R.multiply(order).isInfinity) {
-          val e = BigInt(1, message)
+          val e = BigInt(1, messageHash)
           val rInv = r.modInverse(order)
           //Q = r^(-1)(sR - eG)
           val q = R.multiply(s.bigInteger).subtract(curve.getG.multiply(e.bigInteger)).multiply(rInv.bigInteger)
@@ -131,6 +139,10 @@ object ECDSASignature {
 
 /**
   * ECDSASignature r and s are same as in documentation where signature is represented by tuple (r, s)
+  *
+  * The `publicKey` method is also the way to verify the signature: if the key can be retrieved based
+  * on the signed message, the signature is correct, otherwise it isn't.
+  *
   * @param r - x coordinate of ephemeral public key modulo curve order N
   * @param s - part of the signature calculated with signer private key
   * @param v - public key recovery id
@@ -139,26 +151,26 @@ case class ECDSASignature(r: BigInt, s: BigInt, v: Byte) {
 
   /**
     * returns ECC point encoded with on compression and without leading byte indicating compression
-    * @param message message to be signed
+    * @param messageHash message to be signed; should be a hash of the actual data.
     * @param chainId optional value if you want new signing schema with recovery id calculated with chain id
     * @return
     */
-  def publicKey(message: Array[Byte], chainId: Option[Byte] = None): Option[Array[Byte]] =
-    ECDSASignature.recoverPubBytes(r, s, v, chainId, message)
+  def publicKey(messageHash: Array[Byte], chainId: Option[Byte] = None): Option[Array[Byte]] =
+    ECDSASignature.recoverPubBytes(r, s, v, chainId, messageHash)
 
   /**
     * returns ECC point encoded with on compression and without leading byte indicating compression
-    * @param message message to be signed
+    * @param messageHash message to be signed; should be a hash of the actual data.
     * @return
     */
-  def publicKey(message: ByteString): Option[ByteString] =
-    ECDSASignature.recoverPubBytes(r, s, v, None, message.toArray[Byte]).map(ByteString(_))
+  def publicKey(messageHash: ByteString): Option[ByteString] =
+    ECDSASignature.recoverPubBytes(r, s, v, None, messageHash.toArray[Byte]).map(ByteString(_))
 
   def toBytes: ByteString = {
     import ECDSASignature.RLength
 
     def bigInt2Bytes(b: BigInt) =
-      ByteUtils.padLeft(ByteString(b.toByteArray).takeRight(RLength), RLength, 0)
+      ByteString(ByteUtils.bigIntToBytes(b, RLength))
 
     bigInt2Bytes(r) ++ bigInt2Bytes(s) :+ v
   }

src/main/scala/io/iohk/ethereum/crypto/package.scala

@@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets
 import java.security.SecureRandom
 
 import akka.util.ByteString
+import io.iohk.ethereum.utils.ByteUtils
 import org.bouncycastle.asn1.sec.SECNamedCurves
 import org.bouncycastle.asn1.x9.X9ECParameters
 import org.bouncycastle.crypto.AsymmetricCipherKeyPair
@@ -62,9 +63,11 @@ package object crypto {
     bytes
   }
 
-  /** @return (privateKey, publicKey) pair */
+  /** @return (privateKey, publicKey) pair.
+    * The public key will be uncompressed and have its prefix dropped.
+    */
   def keyPairToByteArrays(keyPair: AsymmetricCipherKeyPair): (Array[Byte], Array[Byte]) = {
-    val prvKey = keyPair.getPrivate.asInstanceOf[ECPrivateKeyParameters].getD.toByteArray
+    val prvKey = ByteUtils.bigIntegerToBytes(keyPair.getPrivate.asInstanceOf[ECPrivateKeyParameters].getD, 32)
     val pubKey = keyPair.getPublic.asInstanceOf[ECPublicKeyParameters].getQ.getEncoded(false).tail
     (prvKey, pubKey)
   }

src/main/scala/io/iohk/ethereum/network/discovery/Secp256k1SigAlg.scala

@@ -0,0 +1,186 @@
+package io.iohk.ethereum.network.discovery
+
+import akka.util.ByteString
+import io.iohk.ethereum.crypto
+import io.iohk.ethereum.crypto.ECDSASignature
+import io.iohk.scalanet.discovery.crypto.{SigAlg, PublicKey, PrivateKey, Signature}
+import io.iohk.ethereum.nodebuilder.SecureRandomBuilder
+import scodec.bits.BitVector
+import scodec.{Attempt, Err}
+import scodec.bits.BitVector
+import org.bouncycastle.crypto.params.ECPublicKeyParameters
+import org.bouncycastle.crypto.AsymmetricCipherKeyPair
+import scala.collection.concurrent.TrieMap
+
+class Secp256k1SigAlg extends SigAlg with SecureRandomBuilder {
+  // We'll be using the same private key over and over to sign messages.
+  // To save the time transforming it into a public-private key pair every time, store the results.
+  // In the future we might want to not pass around the private key but have it as a constructor argument.
+  private val signingKeyPairCache = TrieMap.empty[PrivateKey, AsymmetricCipherKeyPair]
+
+  override val name = "secp256k1"
+
+  override val PrivateKeyBytesSize = 32
+
+  // A Secp256k1 public key is 32 bytes compressed or 64 bytes uncompressed,
+  // with a 1 byte prefix showing which version it is.
+  // See https://davidederosa.com/basic-blockchain-programming/elliptic-curve-keys
+  //
+  // However in the discovery v4 protocol the prefix is omitted.
+  override val PublicKeyBytesSize = 64
+
+  // A normal Secp256k1 signature consists of 2 bigints `r` and `s` followed by a recovery ID `v`,
+  // but it can be just 64 bytes if that's omitted, like in the ENR.
+  override val SignatureBytesSize = 65
+
+  val SignatureWithoutRecoveryBytesSize = 64
+  val PublicKeyCompressedBytesSize = 33
+
+  override def newKeyPair: (PublicKey, PrivateKey) = {
+    val keyPair = crypto.generateKeyPair(secureRandom)
+    val (privateKeyBytes, publicKeyBytes) = crypto.keyPairToByteArrays(keyPair)
+
+    val publicKey = toPublicKey(publicKeyBytes)
+    val privateKey = toPrivateKey(privateKeyBytes)
+
+    publicKey -> privateKey
+  }
+
+  override def sign(privateKey: PrivateKey, data: BitVector): Signature = {
+    val message = crypto.kec256(data.toByteArray)
+    val keyPair = signingKeyPairCache.getOrElseUpdate(privateKey, crypto.keyPairFromPrvKey(privateKey.toByteArray))
+    val sig = ECDSASignature.sign(message, keyPair)
+    toSignature(sig)
+  }
+
+  // ENR wants the signature without recovery ID, just 64 bytes.
+  // The Packet on the other hand has the full 65 bytes.
+  override def removeRecoveryId(signature: Signature): Signature = {
+    signature.size / 8 match {
+      case SignatureBytesSize =>
+        Signature(signature.dropRight(8))
+      case SignatureWithoutRecoveryBytesSize =>
+        signature
+      case other =>
+        throw new IllegalArgumentException(s"Unexpected signature size: $other bytes")
+    }
+  }
+
+  override def compressPublicKey(publicKey: PublicKey): PublicKey = {
+    publicKey.size / 8 match {
+      case PublicKeyBytesSize =>
+        // This is a public key without the prefix, it consists of an x and y bigint.
+        // To compress we drop y, and the first byte becomes 02 for even values of y and 03 for odd values.
+        val point = crypto.curve.getCurve.decodePoint(ECDSASignature.UncompressedIndicator +: publicKey.toByteArray)
+        val key = new ECPublicKeyParameters(point, crypto.curve)
+        val bytes = key.getQ.getEncoded(true) // compressed encoding
+        val compressed = PublicKey(BitVector(bytes))
+        assert(compressed.size == PublicKeyCompressedBytesSize * 8)
+        compressed
+
+      case PublicKeyCompressedBytesSize =>
+        publicKey
+
+      case other =>
+        throw new IllegalArgumentException(s"Unexpected uncompressed public key size: $other bytes")
+    }
+  }
+
+  // The public key points lie on the curve `y^2 = x^3 + 7`.
+  // In the compressed form we have x and a prefix telling us whether y is even or odd.
+  // https://bitcoin.stackexchange.com/questions/86234/how-to-uncompress-a-public-key
+  // https://bitcoin.stackexchange.com/questions/44024/get-uncompressed-public-key-from-compressed-form
+  def decompressPublicKey(publicKey: PublicKey): PublicKey = {
+    publicKey.size / 8 match {
+      case PublicKeyBytesSize =>
+        publicKey
+
+      case PublicKeyCompressedBytesSize =>
+        val point = crypto.curve.getCurve.decodePoint(publicKey.toByteArray)
+        val key = new ECPublicKeyParameters(point, crypto.curve)
+        val bytes = key.getQ.getEncoded(false).drop(1) // uncompressed encoding, drop prefix.
+        toPublicKey(bytes)
+
+      case other =>
+        throw new IllegalArgumentException(s"Unexpected compressed public key size: $other bytes")
+    }
+  }
+
+  override def verify(publicKey: PublicKey, signature: Signature, data: BitVector): Boolean = {
+    val message = crypto.kec256(data.toByteArray)
+    val uncompressedPublicKey = decompressPublicKey(publicKey)
+    toECDSASignatures(signature).exists { sig =>
+      sig.publicKey(message).map(toPublicKey).contains(uncompressedPublicKey)
+    }
+  }
+
+  override def recoverPublicKey(signature: Signature, data: BitVector): Attempt[PublicKey] = {
+    val message = crypto.kec256(data.toByteArray)
+
+    val maybePublicKey = toECDSASignatures(signature).flatMap { sig =>
+      sig.publicKey(message).map(toPublicKey)
+    }.headOption
+
+    Attempt.fromOption(maybePublicKey, Err("Failed to recover the public key from the signature."))
+  }
+
+  override def toPublicKey(privateKey: PrivateKey): PublicKey = {
+    val publicKeyBytes = crypto.pubKeyFromPrvKey(privateKey.toByteArray)
+    toPublicKey(publicKeyBytes)
+  }
+
+  private def toPublicKey(publicKeyBytes: Array[Byte]): PublicKey = {
+    // Discovery uses 64 byte keys, without the prefix.
+    val publicKey = PublicKey(BitVector(publicKeyBytes))
+    assert(publicKey.size == PublicKeyBytesSize * 8, s"Unexpected public key size: ${publicKey.size / 8} bytes")
+    publicKey
+  }
+
+  private def toPrivateKey(privateKeyBytes: Array[Byte]): PrivateKey = {
+    val privateKey = PrivateKey(BitVector(privateKeyBytes))
+    assert(privateKey.size == PrivateKeyBytesSize * 8, s"Unexpected private key size: ${privateKey.size / 8} bytes")
+    privateKey
+  }
+
+  // Apparently the `v` has to adjusted by 27, which is the negative point sign.
+  private def vToWire(v: Byte): Byte =
+    (v - ECDSASignature.negativePointSign).toByte
+
+  private def wireToV(w: Byte): Byte =
+    (w + ECDSASignature.negativePointSign).toByte
+
+  private def adjustV(bytes: Array[Byte], f: Byte => Byte): Unit =
+    bytes(bytes.size - 1) = f(bytes(bytes.size - 1))
+
+  private def toSignature(sig: ECDSASignature): Signature = {
+    val signatureBytes = sig.toBytes.toArray[Byte]
+    assert(signatureBytes.size == SignatureBytesSize)
+    adjustV(signatureBytes, vToWire)
+    Signature(BitVector(signatureBytes))
+  }
+
+  // Based on whether we have the recovery ID in the signature we may have to try 1 or 2 signatures.
+  private def toECDSASignatures(signature: Signature): Iterable[ECDSASignature] = {
+    signature.size / 8 match {
+      case SignatureBytesSize =>
+        val signatureBytes = signature.toByteArray
+        adjustV(signatureBytes, wireToV)
+        Iterable(toECDSASignature(signatureBytes))
+
+      case SignatureWithoutRecoveryBytesSize =>
+        val signatureBytes = signature.toByteArray
+        // Try all allowed points signs.
+        ECDSASignature.allowedPointSigns.toIterable.map { v =>
+          toECDSASignature(signatureBytes :+ v)
+        }
+
+      case other =>
+        throw new IllegalArgumentException(s"Unexpected signature size: $other bytes")
+    }
+  }
+
+  private def toECDSASignature(signatureBytes: Array[Byte]): ECDSASignature =
+    ECDSASignature.fromBytes(ByteString(signatureBytes)) getOrElse {
+      throw new IllegalArgumentException(s"Could not convert to ECDSA signature.")
+    }
+}