Produce charts from benchmark report

Author: julienrf

benchmarks/time/src/main/scala/strawman/collection/mutable/ArrayBufferBenchmark.scala

@@ -3,7 +3,6 @@ package strawman.collection.mutable
 import java.util.concurrent.TimeUnit
 
 import org.openjdk.jmh.annotations._
-import strawman.collection.mutable.ArrayBuffer
 import scala.{Any, AnyRef, Int, Unit}
 
 @BenchmarkMode(scala.Array(Mode.AverageTime))

build.sbt

@@ -26,6 +26,18 @@ val timeBenchmark =
   project.in(file("benchmarks/time"))
     .dependsOn(collections)
     .enablePlugins(JmhPlugin)
+    .settings(
+      // runs the benchmarks and produce charts
+      InputKey[Unit]("charts") := {
+        val benchmarks = Def.spaceDelimited().parsed
+        val targetDir = crossTarget.value / "bencharts"
+        val jmhReport = targetDir / "jmh-result.json"
+        val jmhArgs = s" -rf json -rff ${jmhReport.absolutePath} $benchmarks"
+        // HACK We should use `jmhArgs` here
+        val _ = (run in Jmh).partialInput(" -rf json -rff target/scala-2.12/bencharts/jmh-result.json").evaluated
+        strawman.collection.Bencharts(jmhReport, targetDir)
+      }
+    )
 
 val memoryBenchmark =
   project.in(file("benchmarks/memory"))

project/Bencharts.scala

@@ -0,0 +1,83 @@
+package strawman.collection
+
+import javax.imageio.ImageIO
+
+import org.jfree.chart.JFreeChart
+import org.jfree.chart.axis.{LogAxis, NumberAxis}
+import org.jfree.chart.plot.XYPlot
+import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer
+import org.jfree.data.xy.{XYSeries, XYSeriesCollection}
+import play.api.libs.json.{JsObject, Json}
+import sbt._
+
+object Bencharts {
+
+  /**
+    * Generate charts from the result of a JMH execution.
+    *
+    * Benchmarks that have the same name (e.g. `cons`) are grouped
+    * into a single chart with one series for each.
+    *
+    * @param jmhReport JMH results report
+    * @param targetDir Directory in which the images will be written
+    */
+  def apply(jmhReport: File, targetDir: File): Unit = {
+    val json = Json.parse(IO.read(jmhReport))
+
+    json.as[List[JsObject]]
+      .groupBy { result =>
+        val name = (result \ "benchmark").as[String]
+        val benchmark = name.reverse.takeWhile(_ != '.').reverse
+        benchmark // Benchmark name (e.g. "cons", "foreach", "map")
+      }
+      .foreach { case (benchmark, results) =>
+        val seriess =
+          results
+            // group by concrete collection type
+            .groupBy(result => (result \ "benchmark").as[String].stripSuffix(benchmark))
+            .map { case (collectionType, iterations) =>
+              val xySeries = new XYSeries(collectionType)
+              // each benchmark has been run with several collection sizes (8, 64, 512, etc.)
+              // we add a point for each of these iterations
+              for (iteration <- iterations) {
+                xySeries.add(
+                  (iteration \ "params" \ "size").as[String].toDouble,
+                  (iteration \ "primaryMetric" \ "score").as[Double]
+                )
+              }
+              xySeries
+            }
+
+        val xAxis = new LogAxis("Size")
+        xAxis.setBase(2)
+        xAxis.setStandardTickUnits(NumberAxis.createIntegerTickUnits())
+        val yAxis = new LogAxis("Execution time (lower is better)")
+
+        val col = new XYSeriesCollection()
+        for (series <- seriess) {
+          col.addSeries(series)
+        }
+
+        val plot = new XYPlot(
+          col,
+          xAxis, yAxis,
+          new XYLineAndShapeRenderer(true, true)
+        )
+
+        val chart = new JFreeChart(
+          benchmark,
+          JFreeChart.DEFAULT_TITLE_FONT,
+          plot,
+          true
+        )
+
+        ImageIO.write(
+          chart.createBufferedImage(640, 480),
+          "png",
+          targetDir / s"$benchmark.png"
+        )
+
+      }
+  }
+
+}

project/plugins.sbt

@@ -1 +1,7 @@
-addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.20")
+addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.20")
+
+// for bencharts
+libraryDependencies ++= Seq(
+  "org.jfree" % "jfreechart" % "1.0.14",
+  "com.typesafe.play" %% "play-json" % "2.4.10"
+)