Run warmup phase and allow concurrency configuration

performance-tests/README.md

@@ -80,16 +80,17 @@ Pre-requirements:
 * Have `docker` installed and running - verify by running the `docker` command.
 * Export `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, and `S3_BUCKET` environment variables.
 * By default, each distroConfig runs for 10 seconds. To change this, export `DURATION` environment variable (e.g. `60m`).
+* By default, each distroConfig runs with a concurrency (VUs) of 5. To change this, export `CONCURRENCY` environment variable (e.g. `10`).
 
 Steps:
 * From `aws-otel-python-instrumentation` dir, execute:
 ```sh
 ./scripts/build_and_install_distro.sh
 ./scripts/set-up-performance-tests.sh
 cd performance-tests
-./gradlew test
+./gradlew clean test
 ```
 
 The last step can be run or you can run from IDE (after setting environment variables appropriately).
 
-To diagnose test failures with `./gradlew -i test` or use `-d` for very fine details.
+To diagnose test failures with `./gradlew -i clean test` or use `-d` for very fine details.

performance-tests/src/test/java/io/opentelemetry/OverheadTests.java

@@ -25,10 +25,10 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -108,8 +108,9 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
 
     populateDatabase();
 
-    if (config.getWarmupSeconds() > 0) {
-      // doWarmupPhase(config, vehicleInventoryService);
+    int warmupSeconds = config.getWarmupSeconds();
+    if (warmupSeconds > 0) {
+      doWarmupPhase(warmupSeconds);
     }
 
     long testStart = System.currentTimeMillis();
@@ -142,40 +143,22 @@ private void startRecording(
     vehicleInventoryService.execInContainer(command);
   }
 
-  private void doWarmupPhase(TestConfig testConfig, GenericContainer<?> vehicleInventoryService)
-      throws IOException, InterruptedException {
-    System.out.println(
-        "Performing startup warming phase for " + testConfig.getWarmupSeconds() + " seconds...");
-
-    // excluding the JFR recording from the warmup causes strange inconsistencies in the results
-    System.out.println("Starting disposable JFR warmup recording...");
-    String[] startCommand = {
-      "jcmd",
-      "1",
-      "JFR.start",
-      "settings=/app/overhead.jfc",
-      "dumponexit=true",
-      "name=warmup",
-      "filename=warmup.jfr"
-    };
-    vehicleInventoryService.execInContainer(startCommand);
-
-    long deadline =
-        System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(testConfig.getWarmupSeconds());
-    while (System.currentTimeMillis() < deadline) {
-      GenericContainer<?> k6 =
-          new GenericContainer<>(DockerImageName.parse("loadimpact/k6"))
-              .withNetwork(NETWORK)
-              .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app")
-              .withCommand("run", "-u", "5", "-i", "200", "/app/basic.js")
-              .withStartupCheckStrategy(new OneShotStartupCheckStrategy());
-      k6.start();
+  private void doWarmupPhase(int seconds) {
+    System.out.println("Performing warm up phase for " + seconds + " seconds.");
+    GenericContainer<?> k6 =
+        new GenericContainer<>(DockerImageName.parse("loadimpact/k6"))
+            .withNetwork(NETWORK)
+            .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app")
+            .withCommand("run", "-u", "5", "-d", seconds + "s", "/app/performanceTest.js")
+            .withStartupCheckStrategy(
+                new OneShotStartupCheckStrategy().withTimeout(Duration.ofSeconds(seconds * 2L)));
+    k6.start();
+    sleep(seconds);
+    System.out.println("Awaiting warmup phase end.");
+    while (k6.isRunning()) {
+      System.out.println("Warmup still running.");
+      sleep(1);
     }
-
-    System.out.println("Stopping disposable JFR warmup recording...");
-    String[] stopCommand = {"jcmd", "1", "JFR.stop", "name=warmup"};
-    vehicleInventoryService.execInContainer(stopCommand);
-
     System.out.println("Warmup complete.");
   }
 
@@ -194,4 +177,12 @@ private void writeStartupTimeFile(DistroConfig distroConfig, long start) throws
     Path startupPath = namingConventions.local.startupDurationFile(distroConfig);
     Files.writeString(startupPath, String.valueOf(delta));
   }
+
+  private static void sleep(int seconds) {
+    try {
+      Thread.sleep(seconds * 1000L);
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
+    }
+  }
 }

performance-tests/src/test/java/io/opentelemetry/config/Configs.java

@@ -17,18 +17,20 @@ public enum Configs {
           .name("all-100-tps")
           .description("Compares all DistroConfigs (100TPS test)")
           .withDistroConfigs(DistroConfig.values())
-          .warmupSeconds(60)
+          .warmupSeconds(10)
           .maxRequestRate(100)
           .duration(System.getenv("DURATION"))
+          .concurrentConnections(System.getenv("CONCURRENCY"))
           .build()),
   ALL_800_TPS(
       TestConfig.builder()
           .name("all-800-tps")
           .description("Compares all DistroConfigs (800TPS test)")
           .withDistroConfigs(DistroConfig.values())
-          .warmupSeconds(60)
+          .warmupSeconds(10)
           .maxRequestRate(800)
           .duration(System.getenv("DURATION"))
+          .concurrentConnections(System.getenv("CONCURRENCY"))
           .build());
 
   public final TestConfig config;

performance-tests/src/test/java/io/opentelemetry/config/TestConfig.java

@@ -98,8 +98,10 @@ Builder maxRequestRate(int maxRequestRate) {
       return this;
     }
 
-    Builder concurrentConnections(int concurrentConnections) {
-      this.concurrentConnections = concurrentConnections;
+    Builder concurrentConnections(String concurrentConnections) {
+      if (concurrentConnections != null && !concurrentConnections.isEmpty()) {
+        this.concurrentConnections = Integer.parseInt(concurrentConnections);
+      }
       return this;
     }