provides refactoring of benchmarks (#722)

* moves all the benchmarks to a separate submodule;
* provides an ability to run baseline comparison benches

Signed-off-by: Oleh Dokuka <[email protected]>

Author: OlegDokuka

benchmarks/README.md

@@ -0,0 +1,47 @@
+## Usage of JMH tasks
+
+Only execute specific benchmark(s) (wildcards are added before and after):
+```
+../gradlew jmh --include="(BenchmarkPrimary|OtherBench)"
+```
+If you want to specify the wildcards yourself, you can pass the full regexp:
+```
+../gradlew jmh --fullInclude=.*MyBenchmark.*
+```
+
+Specify extra profilers:
+```
+../gradlew jmh --profilers="gc,stack"
+```
+
+Prominent profilers (for full list call `jmhProfilers` task):
+- comp - JitCompilations, tune your iterations
+- stack - which methods used most time
+- gc - print garbage collection stats
+- hs_thr - thread usage
+
+Change report format from JSON to one of [CSV, JSON, NONE, SCSV, TEXT]:
+```
+./gradlew jmh --format=csv
+```
+
+Specify JVM arguments:
+```
+../gradlew jmh --jvmArgs="-Dtest.cluster=local"
+```
+
+Run in verification mode (execute benchmarks with minimum of fork/warmup-/benchmark-iterations):
+```
+../gradlew jmh --verify=true
+```
+
+## Comparing with the baseline
+If you wish you run two sets of benchmarks, one for the current change and another one for the "baseline",
+there is an additional task `jmhBaseline` that will use the latest release:
+```
+../gradlew jmh jmhBaseline --include=MyBenchmark
+```
+
+## Resources
+- http://tutorials.jenkov.com/java-performance/jmh.html (Introduction)
+- http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/ (Samples)

benchmarks/build.gradle

@@ -0,0 +1,163 @@
+apply plugin: 'java'
+apply plugin: 'idea'
+
+configurations {
+  current
+  baseline {
+    resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
+  }
+}
+
+dependencies {
+  // Use the baseline to avoid using new APIs in the benchmarks
+  compileOnly "io.rsocket:rsocket-core:${perfBaselineVersion}"
+  compileOnly "io.rsocket:rsocket-transport-local:${perfBaselineVersion}"
+
+  implementation "org.openjdk.jmh:jmh-core:1.21"
+  annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:1.21"
+
+  current project(':rsocket-core')
+  current project(':rsocket-transport-local')
+  baseline "io.rsocket:rsocket-core:${perfBaselineVersion}", {
+    changing = true
+  }
+  baseline "io.rsocket:rsocket-transport-local:${perfBaselineVersion}", {
+    changing = true
+  }
+}
+
+task jmhProfilers(type: JavaExec, description:'Lists the available profilers for the jmh task', group: 'Development') {
+  classpath = sourceSets.main.runtimeClasspath
+  main = 'org.openjdk.jmh.Main'
+  args '-lprof'
+}
+
+task jmh(type: JmhExecTask, description: 'Executing JMH benchmarks') {
+  classpath = sourceSets.main.runtimeClasspath + configurations.current
+}
+
+task jmhBaseline(type: JmhExecTask, description: 'Executing JMH baseline benchmarks') {
+  classpath = sourceSets.main.runtimeClasspath + configurations.baseline
+}
+
+class JmhExecTask extends JavaExec {
+
+  private String include;
+  private String fullInclude;
+  private String exclude;
+  private String format = "json";
+  private String profilers;
+  private String jmhJvmArgs;
+  private String verify;
+
+  public JmhExecTask() {
+    super();
+  }
+
+  public String getInclude() {
+    return include;
+  }
+
+  @Option(option = "include", description="configure bench inclusion using substring")
+  public void setInclude(String include) {
+    this.include = include;
+  }
+
+  public String getFullInclude() {
+    return fullInclude;
+  }
+
+  @Option(option = "fullInclude", description = "explicitly configure bench inclusion using full JMH style regexp")
+  public void setFullInclude(String fullInclude) {
+    this.fullInclude = fullInclude;
+  }
+
+  public String getExclude() {
+    return exclude;
+  }
+
+  @Option(option = "exclude", description = "explicitly configure bench exclusion using full JMH style regexp")
+  public void setExclude(String exclude) {
+    this.exclude = exclude;
+  }
+
+  public String getFormat() {
+    return format;
+  }
+
+  @Option(option = "format", description = "configure report format")
+  public void setFormat(String format) {
+    this.format = format;
+  }
+
+  public String getProfilers() {
+    return profilers;
+  }
+
+  @Option(option = "profilers", description = "configure jmh profiler(s) to use, comma separated")
+  public void setProfilers(String profilers) {
+    this.profilers = profilers;
+  }
+
+  public String getJmhJvmArgs() {
+    return jmhJvmArgs;
+  }
+
+  @Option(option = "jvmArgs", description = "configure additional JMH JVM arguments, comma separated")
+  public void setJmhJvmArgs(String jvmArgs) {
+    this.jmhJvmArgs = jvmArgs;
+  }
+
+  public String getVerify() {
+    return verify;
+  }
+
+  @Option(option = "verify", description = "run in verify mode")
+  public void setVerify(String verify) {
+    this.verify = verify;
+  }
+
+  @TaskAction
+  public void exec() {
+    setMain("org.openjdk.jmh.Main");
+    File resultFile = getProject().file("build/reports/" + getName() + "/result." + format);
+
+    if (include != null) {
+      args(".*" + include + ".*");
+    }
+    else if (fullInclude != null) {
+      args(fullInclude);
+    }
+
+    if(exclude != null) {
+      args("-e", exclude);
+    }
+    if(verify != null) { // execute benchmarks with the minimum amount of execution (only to check if they are working)
+      System.out.println("Running in verify mode");
+      args("-f", 1);
+      args("-wi", 1);
+      args("-i", 1);
+    }
+    args("-foe", "true"); //fail-on-error
+    args("-v", "NORMAL"); //verbosity [SILENT, NORMAL, EXTRA]
+    if(profilers != null) {
+      for (String prof : profilers.split(",")) {
+        args("-prof", prof);
+      }
+    }
+    args("-jvmArgsPrepend", "-Xmx3072m");
+    args("-jvmArgsPrepend", "-Xms3072m");
+    if(jmhJvmArgs != null) {
+      for(String jvmArg : jmhJvmArgs.split(" ")) {
+        args("-jvmArgsPrepend", jvmArg);
+      }
+    }
+    args("-rf", format);
+    args("-rff", resultFile);
+
+    System.out.println("\nExecuting JMH with: " + getArgs() + "\n");
+    resultFile.getParentFile().mkdirs();
+
+    super.exec();
+  }
+}

rsocket-core/src/jmh/java/io/rsocket/MaxPerfSubscriber.java renamed to benchmarks/src/main/java/io/rsocket/MaxPerfSubscriber.java

@@ -0,0 +1,16 @@
+package io.rsocket;
+
+import org.openjdk.jmh.infra.Blackhole;
+
+public class PayloadsPerfSubscriber extends PerfSubscriber<Payload> {
+
+  public PayloadsPerfSubscriber(Blackhole blackhole) {
+    super(blackhole);
+  }
+
+  @Override
+  public void onNext(Payload payload) {
+    payload.release();
+    super.onNext(payload);
+  }
+}

benchmarks/src/main/java/io/rsocket/PayloadsPerfSubscriber.java

@@ -5,9 +5,9 @@
 import org.reactivestreams.Subscription;
 import reactor.core.CoreSubscriber;
 
-public class PerfSubscriber implements CoreSubscriber<Payload> {
+public class PerfSubscriber<T> implements CoreSubscriber<T> {
 
-  final CountDownLatch latch = new CountDownLatch(1);
+  public final CountDownLatch latch = new CountDownLatch(1);
   final Blackhole blackhole;
 
   Subscription s;
@@ -23,8 +23,7 @@ public void onSubscribe(Subscription s) {
   }
 
   @Override
-  public void onNext(Payload payload) {
-    payload.release();
+  public void onNext(T payload) {
     blackhole.consume(payload);
     s.request(1);
   }

rsocket-core/src/jmh/java/io/rsocket/PerfSubscriber.java renamed to benchmarks/src/main/java/io/rsocket/PerfSubscriber.java

@@ -4,15 +4,20 @@
 import io.rsocket.transport.local.LocalClientTransport;
 import io.rsocket.transport.local.LocalServerTransport;
 import io.rsocket.util.EmptyPayload;
+import java.lang.reflect.Field;
+import java.util.Queue;
+import java.util.concurrent.locks.LockSupport;
 import java.util.stream.IntStream;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 import org.reactivestreams.Publisher;
@@ -36,11 +41,26 @@ public class RSocketPerf {
 
   RSocket client;
   Closeable server;
+  Queue clientsQueue;
+
+  @TearDown
+  public void tearDown() {
+    client.dispose();
+    server.dispose();
+  }
+
+  @TearDown(Level.Iteration)
+  public void awaitToBeConsumed() {
+    while (!clientsQueue.isEmpty()) {
+      LockSupport.parkNanos(1000);
+    }
+  }
 
   @Setup
-  public void setUp() {
+  public void setUp() throws NoSuchFieldException, IllegalAccessException {
     server =
         RSocketFactory.receive()
+            .frameDecoder(PayloadDecoder.ZERO_COPY)
             .acceptor(
                 (setup, sendingSocket) ->
                     Mono.just(
@@ -75,35 +95,41 @@ public Flux<Payload> requestChannel(Publisher<Payload> payloads) {
 
     client =
         RSocketFactory.connect()
+            .singleSubscriberRequester()
             .frameDecoder(PayloadDecoder.ZERO_COPY)
             .transport(LocalClientTransport.create("server"))
             .start()
             .block();
+
+    Field sendProcessorField = RSocketRequester.class.getDeclaredField("sendProcessor");
+    sendProcessorField.setAccessible(true);
+
+    clientsQueue = (Queue) sendProcessorField.get(client);
   }
 
   @Benchmark
   @SuppressWarnings("unchecked")
-  public PerfSubscriber fireAndForget(Blackhole blackhole) throws InterruptedException {
-    PerfSubscriber subscriber = new PerfSubscriber(blackhole);
+  public PayloadsPerfSubscriber fireAndForget(Blackhole blackhole) throws InterruptedException {
+    PayloadsPerfSubscriber subscriber = new PayloadsPerfSubscriber(blackhole);
     client.fireAndForget(PAYLOAD).subscribe((CoreSubscriber) subscriber);
     subscriber.latch.await();
 
     return subscriber;
   }
 
   @Benchmark
-  public PerfSubscriber requestResponse(Blackhole blackhole) throws InterruptedException {
-    PerfSubscriber subscriber = new PerfSubscriber(blackhole);
+  public PayloadsPerfSubscriber requestResponse(Blackhole blackhole) throws InterruptedException {
+    PayloadsPerfSubscriber subscriber = new PayloadsPerfSubscriber(blackhole);
     client.requestResponse(PAYLOAD).subscribe(subscriber);
     subscriber.latch.await();
 
     return subscriber;
   }
 
   @Benchmark
-  public PerfSubscriber requestStreamWithRequestByOneStrategy(Blackhole blackhole)
+  public PayloadsPerfSubscriber requestStreamWithRequestByOneStrategy(Blackhole blackhole)
       throws InterruptedException {
-    PerfSubscriber subscriber = new PerfSubscriber(blackhole);
+    PayloadsPerfSubscriber subscriber = new PayloadsPerfSubscriber(blackhole);
     client.requestStream(PAYLOAD).subscribe(subscriber);
     subscriber.latch.await();
 
@@ -121,9 +147,9 @@ public MaxPerfSubscriber requestStreamWithRequestAllStrategy(Blackhole blackhole
   }
 
   @Benchmark
-  public PerfSubscriber requestChannelWithRequestByOneStrategy(Blackhole blackhole)
+  public PayloadsPerfSubscriber requestChannelWithRequestByOneStrategy(Blackhole blackhole)
       throws InterruptedException {
-    PerfSubscriber subscriber = new PerfSubscriber(blackhole);
+    PayloadsPerfSubscriber subscriber = new PayloadsPerfSubscriber(blackhole);
     client.requestChannel(PAYLOAD_FLUX).subscribe(subscriber);
     subscriber.latch.await();
 

rsocket-core/src/jmh/java/io/rsocket/RSocketPerf.java renamed to benchmarks/src/main/java/io/rsocket/RSocketPerf.java

@@ -12,3 +12,4 @@
 # limitations under the License.
 #
 version=1.0.0-RC6
+perfBaselineVersion=1.0.0-RC5

rsocket-core/src/jmh/java/io/rsocket/StreamIdSupplierPerf.java renamed to benchmarks/src/main/java/io/rsocket/StreamIdSupplierPerf.java

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.3-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

rsocket-core/src/jmh/java/io/rsocket/frame/FrameHeaderFlyweightPerf.java renamed to benchmarks/src/main/java/io/rsocket/frame/FrameHeaderFlyweightPerf.java

@@ -46,6 +46,4 @@ dependencies {
     testRuntimeOnly 'org.junit.vintage:junit-vintage-engine'
 }
 
-description = "Core functionality for the RSocket library"
-
-apply from: 'jmh.gradle'
+description = "Core functionality for the RSocket library"