[CGKIELE-168] Add support for metrics

Author: Christos KK Loverdos

build.sbt

@@ -55,7 +55,10 @@ val dep = {
     // mallet deps
     "org.jline" % "jline" % "3.1.2",
     "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.5",
-    "com.github.scopt" %% "scopt" % "3.7.0"
+    "com.github.scopt" %% "scopt" % "3.7.0",
+
+    // Metrics (https://github.com/DataDog/java-dogstatsd-client)
+    "com.datadoghq" % "java-dogstatsd-client" % "2.5"
   )
 }
 

src/main/resources/application.conf

@@ -548,6 +548,41 @@ mantis {
     }
   }
 
+  metrics {
+    # Set to `true` iff your deployment supports metrics collection.
+    # We push metrics to a StatsD-compatible agent and we use Datadog for collecting them in one place.
+    # We default to `false` here because we do not expect all deployments to support metrics collection.
+    enabled = false
+
+    # The StatsD-compatible agent host.
+    host = "localhost"
+
+    # The StatsD-compatible agent port.
+    port = 8125
+
+    # The API key of the metrics collector.
+    api-key = ""
+
+    # All metrics sent will be tagged with the environment.
+    # Sample values are: `public`, `private`, `production`, `dev`, depending on the use-case.
+    # If metrics are `enabled`, this is mandatory and must be set explicitly to a non-null value.
+    #
+    # environment = null
+
+    # All metrics sent will be tagged with the deployment.
+    # Sample values are: `kevm-testnet`, `iele-testnet`, `raft-kevm`, depending on the use-case.
+    # If metrics are `enabled`, this is mandatory and must be set explicitly to a non-null value.
+    #
+    # deployment = null
+
+    # Size of the metrics requests queue.
+    # If the queue contains that many outstanding requests to the metrics agent, then
+    # subsequent requests are blocked until the queue has room again.
+    queue-size = 1024
+
+    # Iff true, any errors during metrics client operations will be logged.
+    log-errors = true
+  }
 }
 
 akka {

src/main/scala/io/iohk/ethereum/metrics/Metrics.scala

@@ -0,0 +1,5 @@
+package io.iohk.ethereum.metrics
+
+object Metrics {
+  final val StartCounter = "start.counter"
+}

src/main/scala/io/iohk/ethereum/metrics/MetricsClient.scala

@@ -0,0 +1,54 @@
+package io.iohk.ethereum.metrics
+
+import com.timgroup.statsd.{NoOpStatsDClient, NonBlockingStatsDClient, StatsDClient}
+import com.typesafe.config.Config
+
+
+object MetricsClient {
+  final val Prefix = "mantis" // TODO there are several other strings of this value. Can we consolidate?
+
+  /**
+   * Default tags we send to StatsD (actually Datadog).
+   *
+   * See https://github.com/input-output-hk/iohk-ops/blob/618748e09035f7bc3e3b055818c0cde4cf1958ce/modules/production.nix#L15
+   */
+  object Tag {
+    final val Env = "env"
+    final val Depl = "depl"
+  }
+
+  def mkTag(name: String, value: String): String = s"$name:$value"
+
+  def apply(config: MetricsConfig): StatsDClient = {
+    val prefix = Prefix
+    val enabled = config.enabled
+
+    if(enabled) {
+      val hostname = config.host
+      val port = config.port
+      val queueSize = config.queueSize
+      val logErrors = config.logErrors
+      val constantTags = Array(
+        mkTag(Tag.Env, config.environment),
+        mkTag(Tag.Depl, config.deployment)
+      )
+      val errorHandler = if(logErrors) new MetricsErrorHandler else null // null indicates NOOP
+
+      new NonBlockingStatsDClient(
+        prefix,
+        hostname,
+        port,
+        queueSize,
+        constantTags,
+        errorHandler
+      )
+    } else {
+      new NoOpStatsDClient
+    }
+  }
+
+  def apply(mantisConfig: Config): StatsDClient = {
+    val config = MetricsConfig(mantisConfig)
+    this(config)
+  }
+}

src/main/scala/io/iohk/ethereum/metrics/MetricsConfig.scala

@@ -0,0 +1,55 @@
+package io.iohk.ethereum.metrics
+
+import com.typesafe.config.{Config ⇒ TypesafeConfig}
+
+final case class MetricsConfig(
+  enabled: Boolean,
+  host: String,
+  port: Int,
+  queueSize: Int,
+  logErrors: Boolean,
+  environment: String, // `public`, `private`
+  deployment: String,  // `testnet-kevm`, `testnet-iele`
+  apiKey: String
+)
+
+object MetricsConfig {
+  object Keys {
+    final val Metrics = "metrics"
+
+    final val Enabled = "enabled"
+    final val Host = "host"
+    final val Port = "port"
+    final val QueueSize = "queue-size"
+    final val LogErrors = "log-errors"
+
+    final val Environment = "environment"
+    final val Deployment = "deployment"
+    final val ApiKey = "api-key"
+  }
+
+  def apply(mantisConfig: TypesafeConfig): MetricsConfig = {
+    val config = mantisConfig.getConfig(Keys.Metrics)
+
+    val enabled = config.getBoolean(Keys.Enabled)
+    val host = config.getString(Keys.Host)
+    val port = config.getInt(Keys.Port)
+    val queueSize = config.getInt(Keys.QueueSize)
+    val logErrors = config.getBoolean(Keys.LogErrors)
+
+    val environment = config.getString(Keys.Environment)
+    val deployment = config.getString(Keys.Deployment)
+    val apiKey = config.getString(Keys.ApiKey)
+
+    MetricsConfig(
+      enabled = enabled,
+      host = host,
+      port = port,
+      queueSize = queueSize,
+      logErrors = logErrors,
+      environment = environment,
+      deployment = environment,
+      apiKey = apiKey
+    )
+  }
+}

src/main/scala/io/iohk/ethereum/metrics/MetricsErrorHandler.scala

@@ -0,0 +1,9 @@
+package io.iohk.ethereum.metrics
+
+import com.timgroup.statsd.StatsDClientErrorHandler
+import io.iohk.ethereum.utils.Logger
+
+final class MetricsErrorHandler extends StatsDClientErrorHandler with Logger {
+  def handle(exception: Exception): Unit =
+    log.error("[StatsDClientErrorHandler]", exception)
+}

src/main/scala/io/iohk/ethereum/nodebuilder/NodeBuilder.scala

@@ -5,6 +5,7 @@ import java.time.Clock
 
 import akka.actor.{ActorRef, ActorSystem}
 import akka.agent.Agent
+import com.timgroup.statsd.StatsDClient
 import io.iohk.ethereum.blockchain.data.GenesisDataLoader
 import io.iohk.ethereum.blockchain.sync.{BlockchainHostActor, SyncController}
 import io.iohk.ethereum.consensus._
@@ -22,6 +23,7 @@ import io.iohk.ethereum.jsonrpc.server.http.JsonRpcHttpServer
 import io.iohk.ethereum.jsonrpc.server.ipc.JsonRpcIpcServer
 import io.iohk.ethereum.keystore.{KeyStore, KeyStoreImpl}
 import io.iohk.ethereum.ledger.Ledger.VMImpl
+import io.iohk.ethereum.metrics.MetricsClient
 import io.iohk.ethereum.network.PeerManagerActor.PeerConfiguration
 import io.iohk.ethereum.network.EtcPeerManagerActor.PeerInfo
 import io.iohk.ethereum.utils._
@@ -442,6 +444,12 @@ trait SyncControllerBuilder {
 
 }
 
+trait MetricsClientBuilder {
+  protected def buildMetricsClient(): StatsDClient = MetricsClient(Config.config)
+
+  lazy val metricsClient: StatsDClient = buildMetricsClient()
+}
+
 trait ShutdownHookBuilder { self: Logger ⇒
   def shutdown(): Unit = {/* No default behaviour during shutdown. */}
 
@@ -531,3 +539,4 @@ trait Node extends NodeKeyBuilder
   with ConsensusBuilder
   with ConsensusConfigBuilder
   with LedgerBuilder
+  with MetricsClientBuilder

src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala

@@ -2,6 +2,7 @@ package io.iohk.ethereum.nodebuilder
 
 import io.iohk.ethereum.blockchain.sync.SyncController
 import io.iohk.ethereum.consensus.StdConsensusBuilder
+import io.iohk.ethereum.metrics.Metrics
 import io.iohk.ethereum.network.{PeerManagerActor, ServerActor}
 import io.iohk.ethereum.network.discovery.DiscoveryListener
 import io.iohk.ethereum.testmode.{TestLedgerBuilder, TestmodeConsensusBuilder}
@@ -50,6 +51,11 @@ abstract class BaseNode extends Node {
     if (jsonRpcConfig.ipcServerConfig.enabled) jsonRpcIpcServer.run()
   }
 
+  private[this] def recordStartMetrics(): Unit = {
+    // How many times has this node been started?
+    metricsClient.incrementCounter(Metrics.StartCounter)
+  }
+
   def start(): Unit = {
     loadGenesisData()
 
@@ -67,6 +73,8 @@ abstract class BaseNode extends Node {
 
     startJsonRpcHttpServer()
     startJsonRpcIpcServer()
+
+    recordStartMetrics()
   }
 
   override def shutdown(): Unit = {
@@ -81,6 +89,7 @@ abstract class BaseNode extends Node {
     if (jsonRpcConfig.ipcServerConfig.enabled) {
       tryAndLogFailure(() => jsonRpcIpcServer.close())
     }
+    tryAndLogFailure(() => metricsClient.close())
   }
 }
 

src/universal/conf/mantis.conf

@@ -14,6 +14,7 @@ include "blockchain.conf"
 include "sync.conf"
 include "misc.conf"
 include "consensus.conf"
+include "metrics.conf"
 
 # Uncomment the following include to connect to the Ethereum Hard-Fork network.
 # Note that any settings in this file will override the ones defined in the files above.

src/universal/conf/metrics.conf

@@ -0,0 +1,37 @@
+mantis {
+  metrics {
+    # Set to `true` iff your deployment supports metrics collection.
+    # We push metrics to a StatsD-compatible agent and we use Datadog for collecting them in one place.
+    # We default to `false` here because we do not expect all deployments to support metrics collection.
+    # enabled = false
+
+    # The StatsD-compatible agent host.
+    # host = "localhost"
+
+    # The StatsD-compatible agent port.
+    # port = 8125
+
+    # The API key of the metrics collector.
+    # api-key = ""
+
+    # All metrics sent will be tagged with the environment.
+    # Sample values are: `public`, `private`, `production`, `dev`, depending on the use-case.
+    # If metrics are `enabled`, this is mandatory and must be set explicitly to a non-null value.
+    #
+    # environment = null
+
+    # All metrics sent will be tagged with the deployment.
+    # Sample values are: `kevm-testnet`, `iele-testnet`, `raft-kevm`, depending on the use-case.
+    # If metrics are `enabled`, this is mandatory and must be set explicitly to a non-null value.
+    #
+    # deployment = null
+
+    # Size of the metrics requests queue.
+    # If the queue contains that many outstanding requests to the metrics agent, then
+    # subsequent requests are blocked until the queue has room again.
+    # queue-size = 1024
+
+    # Iff true, any errors during metrics client operations will be logged.
+    # log-errors = true
+  }
+}

verify.sbt

@@ -89,7 +89,8 @@ verifyDependencies in verify ++= Seq(
   "io.atomix" % "atomix-storage" sha1 "136f0b221acbc2680f099b8ff3a34f8cc1592fe7",
   "io.atomix" % "atomix-primary-backup" sha1 "1c895965e3e67a152ffbccb4283b6cee91b4ea61",
   "io.netty" % "netty-tcnative-boringssl-static" sha1 "ff5f2d6db5aaa1b4df1b381382cd6581844aad9d",
-  "com.github.scopt" % "scopt" sha1 "e078455e1a65597146f8608dab3247bf1eb92e6e"
+  "com.github.scopt" % "scopt" sha1 "e078455e1a65597146f8608dab3247bf1eb92e6e",
+  "com.datadoghq" % "java-dogstatsd-client" sha1 "a9380127a42855a76af7787840a3a04b9fc4ce20"
 )
 
 verifyOptions in verify := VerifyOptions(