Skip to content

Commit 8ae3eef

Browse files
pslaskipslaski
authored
Merge pull request #718 from input-output-hk/etcm-78-checkpointing-jrc
[ETCM-78 added checkpointing JRC
2 parents 946b711 + e6f4714 commit 8ae3eef

File tree

15 files changed

+596
-45
lines changed

15 files changed

+596
-45
lines changed

‎build.sbt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ scalacOptions := Seq(
8585
"-Xlint:unsound-match",
8686
"-Ywarn-inaccessible",
8787
"-Ywarn-unused-import",
88+
"-Ypartial-unification",
8889
"-encoding",
8990
"utf-8"
9091
)

‎insomnia_workspace.json

Lines changed: 96 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,95 @@
11
{
22
"_type": "export",
33
"__export_format": 4,
4-
"__export_date": "2020-09-30T20:01:40.792Z",
4+
"__export_date": "2020-10-02T11:18:34.806Z",
55
"__export_source": "insomnia.desktop.app:v2020.4.1",
66
"resources": [
7+
{
8+
"_id": "req_4222a4d54ba24fa7813429bdcdb732df",
9+
"parentId": "fld_a7212a5b96194230a7e0abc76ee2bf26",
10+
"modified": 1601637263922,
11+
"created": 1601637164399,
12+
"url": "{{ node_url }}",
13+
"name": "checkpointing_getLatestBlock",
14+
"description": "",
15+
"method": "POST",
16+
"body": {
17+
"mimeType": "application/json",
18+
"text": "{\n\t\"jsonrpc\": \"2.0\",\n\t\"id\": 1,\n\t\"method\": \"checkpointing_getLatestBlock\",\n\t\"params\": [5]\n}"
19+
},
20+
"parameters": [],
21+
"headers": [
22+
{
23+
"name": "Content-Type",
24+
"value": "application/json",
25+
"id": "pair_a28efff3b44442d99b7f3fbc54d8c9b6"
26+
}
27+
],
28+
"authentication": {},
29+
"metaSortKey": -1601637164399,
30+
"isPrivate": false,
31+
"settingStoreCookies": true,
32+
"settingSendCookies": true,
33+
"settingDisableRenderRequestBody": false,
34+
"settingEncodeUrl": true,
35+
"settingRebuildPath": true,
36+
"settingFollowRedirects": "global",
37+
"_type": "request"
38+
},
39+
{
40+
"_id": "fld_a7212a5b96194230a7e0abc76ee2bf26",
41+
"parentId": "wrk_097d43914a4d4aea8b6f73f647921182",
42+
"modified": 1601637156313,
43+
"created": 1601637156313,
44+
"name": "Checkpointing",
45+
"description": "",
46+
"environment": {},
47+
"environmentPropertyOrder": null,
48+
"metaSortKey": -1601637156313,
49+
"_type": "request_group"
50+
},
51+
{
52+
"_id": "wrk_097d43914a4d4aea8b6f73f647921182",
53+
"parentId": null,
54+
"modified": 1599825617921,
55+
"created": 1552662762769,
56+
"name": "Mantis",
57+
"description": "",
58+
"scope": null,
59+
"_type": "workspace"
60+
},
61+
{
62+
"_id": "req_da1e409360394849b673ec4e27f542b6",
63+
"parentId": "fld_a7212a5b96194230a7e0abc76ee2bf26",
64+
"modified": 1601637193229,
65+
"created": 1601637186866,
66+
"url": "{{ node_url }}",
67+
"name": "checkpointing_pushCheckpoint",
68+
"description": "",
69+
"method": "POST",
70+
"body": {
71+
"mimeType": "application/json",
72+
"text": "{\n\t\"jsonrpc\": \"2.0\",\n\t\"id\": 1,\n\t\"method\": \"checkpointing_pushCheckpoint\",\n\t\"params\": [\n\t\t\"127d6fde40d20208641c057a1ad4d12d44433881a660b15ac99f04f25762fb9b\",\n\t\t[\n\t\"2194b40851c648e7570e75ea2c507887d11c2270f7523469953fc5c3d5e0f50f48d73ea0b827eb81bb2fc0511f09d10b8f1d3f88e251ed231bb0f5cd03826d281b\",\n\t\t\t\"bbd4ae567202a6e7f40826c964a918760253596bb92052ea7ef4b30338b19fc12d56d497c88f0f13eff0ad542a8a4c1069559cb43e9741b849bf6577287450e31b\"\n\t\t]\n\t]\n}"
73+
},
74+
"parameters": [],
75+
"headers": [
76+
{
77+
"name": "Content-Type",
78+
"value": "application/json",
79+
"id": "pair_a28efff3b44442d99b7f3fbc54d8c9b6"
80+
}
81+
],
82+
"authentication": {},
83+
"metaSortKey": -1583661254601.5,
84+
"isPrivate": false,
85+
"settingStoreCookies": true,
86+
"settingSendCookies": true,
87+
"settingDisableRenderRequestBody": false,
88+
"settingEncodeUrl": true,
89+
"settingRebuildPath": true,
90+
"settingFollowRedirects": "global",
91+
"_type": "request"
92+
},
793
{
894
"_id": "req_cd0078ce4a034ebdbdf7dc9e20e78a29",
995
"parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
@@ -53,16 +139,6 @@
53139
"metaSortKey": -1600249374160,
54140
"_type": "request_group"
55141
},
56-
{
57-
"_id": "wrk_097d43914a4d4aea8b6f73f647921182",
58-
"parentId": null,
59-
"modified": 1599825617921,
60-
"created": 1552662762769,
61-
"name": "Mantis",
62-
"description": "",
63-
"scope": null,
64-
"_type": "workspace"
65-
},
66142
{
67143
"_id": "req_6197fefa1e1448a89f30712ec12295f8",
68144
"parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
@@ -1034,12 +1110,18 @@
10341110
"modified": 1599825641645,
10351111
"created": 1552663140073,
10361112
"name": "Develop",
1037-
"data": { "node_url": "http://127.0.0.1:8546" },
1038-
"dataPropertyOrder": { "&": ["node_url"] },
1113+
"data": {
1114+
"node_url": "http://127.0.0.1:8546"
1115+
},
1116+
"dataPropertyOrder": {
1117+
"&": [
1118+
"node_url"
1119+
]
1120+
},
10391121
"color": null,
10401122
"isPrivate": false,
10411123
"metaSortKey": 1552663140073,
10421124
"_type": "environment"
10431125
}
10441126
]
1045-
}
1127+
}

‎src/main/resources/application.conf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -185,8 +185,8 @@ mantis {
185185
}
186186

187187
# Enabled JSON-RPC APIs over the JSON-RPC endpoint
188-
# Available choices are: eth, web3, net, personal, test, daedalus, iele, qa
189-
apis = "eth,web3,net,personal,daedalus,debug,qa"
188+
# Available choices are: web3, eth, net, personal, daedalus, test, iele, debug, qa, checkpointing
189+
apis = "eth,web3,net,personal,daedalus,debug,qa,checkpointing"
190190

191191
# Maximum number of blocks for daedalus_getAccountTransactions
192192
account-transactions-max-blocks = 50000

‎src/main/scala/io/iohk/ethereum/blockchain/sync/regular/RegularSync.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package io.iohk.ethereum.blockchain.sync.regular
22

33
import akka.actor.{Actor, ActorLogging, ActorRef, AllForOneStrategy, Cancellable, Props, Scheduler, SupervisorStrategy}
4+
import akka.util.ByteString
45
import io.iohk.ethereum.blockchain.sync.BlockBroadcast
6+
import io.iohk.ethereum.crypto.ECDSASignature
57
import io.iohk.ethereum.domain.{Block, Blockchain}
68
import io.iohk.ethereum.ledger.Ledger
79
import io.iohk.ethereum.utils.Config.SyncConfig
@@ -95,4 +97,5 @@ object RegularSync {
9597
sealed trait RegularSyncMsg
9698
case object Start extends RegularSyncMsg
9799
case class MinedBlock(block: Block) extends RegularSyncMsg
100+
case class NewCheckpoint(parentHash: ByteString, signatures: Seq[ECDSASignature]) extends RegularSyncMsg
98101
}

‎src/main/scala/io/iohk/ethereum/jsonrpc/CheckpointingJsonMethodsImplicits.scala

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package io.iohk.ethereum.jsonrpc
2+
3+
import akka.util.ByteString
4+
import io.iohk.ethereum.crypto.ECDSASignature
5+
import io.iohk.ethereum.jsonrpc.CheckpointingService._
6+
import io.iohk.ethereum.jsonrpc.JsonRpcController.Codec
7+
import io.iohk.ethereum.jsonrpc.JsonRpcErrors.InvalidParams
8+
import io.iohk.ethereum.jsonrpc.JsonSerializers.QuantitiesSerializer
9+
import org.json4s.JsonAST._
10+
import org.json4s.{Extraction, JsonAST}
11+
12+
object CheckpointingJsonMethodsImplicits extends JsonMethodsImplicits {
13+
14+
implicit val checkpointing_getLatestBlock: Codec[GetLatestBlockRequest, GetLatestBlockResponse] =
15+
new Codec[GetLatestBlockRequest, GetLatestBlockResponse] {
16+
override def decodeJson(
17+
params: Option[JsonAST.JArray]
18+
): Either[JsonRpcError, GetLatestBlockRequest] =
19+
params match {
20+
case Some(JArray(JInt(chkpInterval) :: Nil)) =>
21+
if (chkpInterval > 0 && chkpInterval <= Int.MaxValue)
22+
Right(GetLatestBlockRequest(chkpInterval.toInt))
23+
else
24+
Left(InvalidParams("Expected positive integer"))
25+
case _ =>
26+
Left(InvalidParams())
27+
}
28+
29+
override def encodeJson(resp: GetLatestBlockResponse): JsonAST.JValue =
30+
Extraction.decompose(resp)(formats - QuantitiesSerializer)
31+
}
32+
33+
implicit val checkpointing_pushCheckpoint: Codec[PushCheckpointRequest, PushCheckpointResponse] =
34+
new Codec[PushCheckpointRequest, PushCheckpointResponse] {
35+
override def decodeJson(
36+
params: Option[JsonAST.JArray]
37+
): Either[JsonRpcError, PushCheckpointRequest] =
38+
params match {
39+
case Some(JArray(JString(hashStr) :: (signatures: JArray) :: Nil)) =>
40+
for {
41+
hash <- extractHash(hashStr)
42+
sigs <- extractSignatures(signatures)
43+
} yield PushCheckpointRequest(hash, sigs)
44+
45+
case _ =>
46+
Left(InvalidParams())
47+
}
48+
49+
override def encodeJson(t: PushCheckpointResponse): JsonAST.JValue =
50+
JBool(true)
51+
}
52+
53+
private def extractSignatures(arr: JArray): Either[JsonRpcError, List[ECDSASignature]] = {
54+
import cats.implicits._
55+
def parseSig(bs: ByteString) =
56+
ECDSASignature.fromBytes(bs).toRight(InvalidParams("Bad signature length"))
57+
58+
arr.arr.traverse {
59+
case JString(str) => extractBytes(str).flatMap(parseSig)
60+
61+
case other => Left(InvalidParams(s"Unable to extract a signature from: $other"))
62+
}
63+
}
64+
}

‎src/main/scala/io/iohk/ethereum/jsonrpc/CheckpointingService.scala

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package io.iohk.ethereum.jsonrpc
2+
3+
import akka.actor.ActorRef
4+
import akka.util.ByteString
5+
import io.iohk.ethereum.blockchain.sync.regular.RegularSync.NewCheckpoint
6+
import io.iohk.ethereum.crypto.ECDSASignature
7+
import io.iohk.ethereum.domain.Blockchain
8+
import io.iohk.ethereum.utils.Logger
9+
import monix.execution.Scheduler.Implicits.global
10+
11+
import scala.concurrent.Future
12+
13+
class CheckpointingService(
14+
blockchain: Blockchain,
15+
syncController: ActorRef
16+
) extends Logger {
17+
18+
import CheckpointingService._
19+
20+
def getLatestBlock(req: GetLatestBlockRequest): ServiceResponse[GetLatestBlockResponse] = {
21+
lazy val bestBlockNum = blockchain.getBestBlockNumber()
22+
lazy val blockToReturnNum = bestBlockNum - bestBlockNum % req.checkpointingInterval
23+
24+
Future {
25+
blockchain.getBlockByNumber(blockToReturnNum)
26+
}.flatMap {
27+
case Some(b) =>
28+
val resp = GetLatestBlockResponse(b.hash, b.number)
29+
Future.successful(Right(resp))
30+
31+
case None =>
32+
log.error(
33+
s"Failed to retrieve block for checkpointing: block at number $blockToReturnNum was unavailable " +
34+
s"even though best block number was $bestBlockNum (re-org occurred?)"
35+
)
36+
getLatestBlock(req) // this can fail only during a re-org, so we just try again
37+
}
38+
}
39+
40+
def pushCheckpoint(req: PushCheckpointRequest): ServiceResponse[PushCheckpointResponse] = Future {
41+
syncController ! NewCheckpoint(req.hash, req.signatures)
42+
Right(PushCheckpointResponse())
43+
}
44+
}
45+
46+
object CheckpointingService {
47+
case class GetLatestBlockRequest(checkpointingInterval: Int)
48+
case class GetLatestBlockResponse(hash: ByteString, number: BigInt)
49+
50+
case class PushCheckpointRequest(hash: ByteString, signatures: List[ECDSASignature])
51+
case class PushCheckpointResponse()
52+
}

‎src/main/scala/io/iohk/ethereum/jsonrpc/JsonMethodsImplicits.scala

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,7 @@ import io.iohk.ethereum.jsonrpc.EthService.BlockParam
99
import io.iohk.ethereum.jsonrpc.JsonRpcController.JsonDecoder.NoParamsDecoder
1010
import io.iohk.ethereum.jsonrpc.JsonRpcController.{Codec, JsonDecoder, JsonEncoder}
1111
import io.iohk.ethereum.jsonrpc.JsonRpcErrors.InvalidParams
12-
import io.iohk.ethereum.jsonrpc.JsonSerializers.{
13-
AddressJsonSerializer,
14-
OptionNoneToJNullSerializer,
15-
QuantitiesSerializer,
16-
UnformattedDataJsonSerializer
17-
}
12+
import io.iohk.ethereum.jsonrpc.JsonSerializers.{AddressJsonSerializer, OptionNoneToJNullSerializer, QuantitiesSerializer, UnformattedDataJsonSerializer}
1813
import io.iohk.ethereum.jsonrpc.NetService._
1914
import io.iohk.ethereum.jsonrpc.PersonalService._
2015
import io.iohk.ethereum.jsonrpc.Web3Service.{ClientVersionRequest, ClientVersionResponse, Sha3Request, Sha3Response}

‎src/main/scala/io/iohk/ethereum/jsonrpc/JsonRpcController.scala

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ import io.iohk.ethereum.jsonrpc.TestService._
1616
import io.iohk.ethereum.jsonrpc.server.http.JsonRpcHttpServer.JsonRpcHttpServerConfig
1717
import io.iohk.ethereum.jsonrpc.server.ipc.JsonRpcIpcServer.JsonRpcIpcServerConfig
1818
import java.util.concurrent.TimeUnit
19+
20+
import io.iohk.ethereum.jsonrpc.CheckpointingService._
21+
1922
import scala.concurrent.Future
2023
import scala.concurrent.ExecutionContext.Implicits.global
2124
import scala.concurrent.duration.FiniteDuration
@@ -69,7 +72,7 @@ object JsonRpcController {
6972
override val apis: Seq[String] = {
7073
val providedApis = rpcConfig.getString("apis").split(",").map(_.trim.toLowerCase)
7174
val invalidApis =
72-
providedApis.diff(List("web3", "eth", "net", "personal", "daedalus", "test", "iele", "debug", "qa"))
75+
providedApis.diff(Apis.available)
7376
require(invalidApis.isEmpty, s"Invalid RPC APIs specified: ${invalidApis.mkString(",")}")
7477
providedApis
7578
}
@@ -87,15 +90,16 @@ object JsonRpcController {
8790
val Eth = "eth"
8891
val Web3 = "web3"
8992
val Net = "net"
90-
val Db = "db"
9193
val Personal = "personal"
9294
val Daedalus = "daedalus"
93-
val Admin = "admin"
9495
val Debug = "debug"
9596
val Rpc = "rpc"
9697
val Test = "test"
9798
val Iele = "iele"
9899
val Qa = "qa"
100+
val Checkpointing = "checkpointing"
101+
102+
val available = Seq(Eth, Web3, Net, Personal, Daedalus, Debug, Test, Iele, Qa, Checkpointing)
99103
}
100104

101105
}
@@ -108,6 +112,7 @@ class JsonRpcController(
108112
testServiceOpt: Option[TestService],
109113
debugService: DebugService,
110114
qaService: QAService,
115+
checkpointingService: CheckpointingService,
111116
config: JsonRpcConfig
112117
) extends Logger {
113118

@@ -119,20 +124,20 @@ class JsonRpcController(
119124
import JsonRpcErrors._
120125
import DebugJsonMethodsImplicits._
121126
import QAJsonMethodsImplicits._
127+
import CheckpointingJsonMethodsImplicits._
122128

123129
lazy val apisHandleFns: Map[String, PartialFunction[JsonRpcRequest, Future[JsonRpcResponse]]] = Map(
124130
Apis.Eth -> handleEthRequest,
125131
Apis.Web3 -> handleWeb3Request,
126132
Apis.Net -> handleNetRequest,
127-
Apis.Db -> PartialFunction.empty,
128133
Apis.Personal -> handlePersonalRequest,
129134
Apis.Daedalus -> handleDaedalusRequest,
130135
Apis.Rpc -> handleRpcRequest,
131-
Apis.Admin -> PartialFunction.empty,
132136
Apis.Debug -> handleDebugRequest,
133137
Apis.Test -> handleTestRequest,
134138
Apis.Iele -> handleIeleRequest,
135-
Apis.Qa -> handleQARequest
139+
Apis.Qa -> handleQARequest,
140+
Apis.Checkpointing -> handleCheckpointingRequest
136141
)
137142

138143
private def enabledApis: Seq[String] = config.apis :+ Apis.Rpc // RPC enabled by default
@@ -349,6 +354,14 @@ class JsonRpcController(
349354
handle[GetPendingTransactionsRequest, GetPendingTransactionsResponse](qaService.getPendingTransactions, req)
350355
}
351356

357+
private def handleCheckpointingRequest: PartialFunction[JsonRpcRequest, Future[JsonRpcResponse]] = {
358+
case req @ JsonRpcRequest(_, "checkpointing_getLatestBlock", _, _) =>
359+
handle[GetLatestBlockRequest, GetLatestBlockResponse](checkpointingService.getLatestBlock, req)
360+
361+
case req @ JsonRpcRequest(_, "checkpointing_pushCheckpoint", _, _) =>
362+
handle[PushCheckpointRequest, PushCheckpointResponse](checkpointingService.pushCheckpoint, req)
363+
}
364+
352365
private def handleRpcRequest: PartialFunction[JsonRpcRequest, Future[JsonRpcResponse]] = {
353366
case req @ JsonRpcRequest(_, "rpc_modules", _, _) =>
354367
val result = enabledApis.map { _ -> "1.0" }.toMap

0 commit comments

Comments
 (0)