Run warmup phase and allow concurrency configuration (#95)

In this commit we are enabling the warmup phase and allowing runs to
have configurable concurrency. Also improved the commands provided in
the README.

Testing:
```
OverheadTests > runAllTestConfigurations() > all-800-tps STANDARD_OUT
    ----------------------------------------------------------
     Run at Mon Mar 04 20:42:29 UTC 2024
     all-800-tps : Compares all DistroConfigs (800TPS test)
     100 users, 1s duration
    ----------------------------------------------------------
    DistroConfig                  :                      none     app_signals_disabled    app_signals_no_traces       app_signals_traces
    Run duration                  :                  00:00:13                 00:00:11                 00:00:13                 00:00:11
    Avg. CPU (user) %             :                       0.0                      0.0                      0.0                      0.0
    Max. CPU (user) %             :                       0.0                      0.0                      0.0                      0.0
    Avg. mch tot cpu %            :                       0.0                      0.0                      0.0                      0.0
    Startup time (ms)             :                      4010                     5013                     5014                     5013
    Total allocated MB            :                      0.00                     0.00                     0.00                     0.00
    Thread switch rate            :                       0.0                      0.0                      0.0                      0.0
    GC time (ms)                  :                         0                        0                        0                        0
    GC pause time (ms)            :                         0                        0                        0                        0
    Req. Count                    :                   1400.00                  1400.00                  1400.00                  1400.00
    Req. Rate                     :                    124.01                   144.99                   119.48                   146.53
    Req. Lat. mean (ms)           :                    481.05                   462.39                   509.49                   504.60
    Req. Lat. p0 (ms)             :                      2.06                     2.85                     2.08                     4.23
    Req. Lat. p50 (ms)            :                    159.19                   190.15                   175.91                   200.18
    Req. Lat. p90 (ms)            :                   1143.15                  1303.96                  1391.22                  1427.85
    Req. Lat. p99 (ms)            :                   5058.14                  3840.13                  4229.97                  4261.41
    Req. Lat. p100 (ms)           :                   8397.74                  5530.18                  8578.33                  5468.59
    Net read avg (bps)            :                      0.00                     0.00                     0.00                     0.00
    Net write avg (bps)           :                      0.00                     0.00                     0.00                     0.00
    Peak threads                  :                         0                        0                        0                        0

Gradle Test Executor 41 finished executing tests.

> Task :test
Finished generating test XML results (0.145 secs) into: /workplace/thp/python-sdk/aws-otel-python-instrumentation/performance-tests/build/test-results/test
Generating HTML test report...
Finished generating test html results (0.106 secs) into: /workplace/thp/python-sdk/aws-otel-python-instrumentation/performance-tests/build/reports/tests/test

Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.

You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.

For more on this, please refer to https://docs.gradle.org/8.6/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.

BUILD SUCCESSFUL in 6m 49s
4 actionable tasks: 4 executed
```

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: thpierce

performance-tests/README.md

@@ -80,16 +80,17 @@ Pre-requirements:
 * Have `docker` installed and running - verify by running the `docker` command.
 * Export `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, and `S3_BUCKET` environment variables.
 * By default, each distroConfig runs for 10 seconds. To change this, export `DURATION` environment variable (e.g. `60m`).
+* By default, each distroConfig runs with a concurrency (VUs) of 5. To change this, export `CONCURRENCY` environment variable (e.g. `10`).
 
 Steps:
 * From `aws-otel-python-instrumentation` dir, execute:
 ```sh
 ./scripts/build_and_install_distro.sh
 ./scripts/set-up-performance-tests.sh
 cd performance-tests
-./gradlew test
+./gradlew clean test
 ```
 
 The last step can be run or you can run from IDE (after setting environment variables appropriately).
 
-To diagnose test failures with `./gradlew -i test` or use `-d` for very fine details.
+To diagnose test failures with `./gradlew -i clean test` or use `-d` for very fine details.

performance-tests/src/test/java/io/opentelemetry/OverheadTests.java

@@ -25,10 +25,10 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -108,8 +108,9 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
 
     populateDatabase();
 
-    if (config.getWarmupSeconds() > 0) {
-      // doWarmupPhase(config, vehicleInventoryService);
+    int warmupSeconds = config.getWarmupSeconds();
+    if (warmupSeconds > 0) {
+      doWarmupPhase(warmupSeconds);
     }
 
     long testStart = System.currentTimeMillis();
@@ -142,40 +143,22 @@ private void startRecording(
     vehicleInventoryService.execInContainer(command);
   }
 
-  private void doWarmupPhase(TestConfig testConfig, GenericContainer<?> vehicleInventoryService)
-      throws IOException, InterruptedException {
-    System.out.println(
-        "Performing startup warming phase for " + testConfig.getWarmupSeconds() + " seconds...");
-
-    // excluding the JFR recording from the warmup causes strange inconsistencies in the results
-    System.out.println("Starting disposable JFR warmup recording...");
-    String[] startCommand = {
-      "jcmd",
-      "1",
-      "JFR.start",
-      "settings=/app/overhead.jfc",
-      "dumponexit=true",
-      "name=warmup",
-      "filename=warmup.jfr"
-    };
-    vehicleInventoryService.execInContainer(startCommand);
-
-    long deadline =
-        System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(testConfig.getWarmupSeconds());
-    while (System.currentTimeMillis() < deadline) {
-      GenericContainer<?> k6 =
-          new GenericContainer<>(DockerImageName.parse("loadimpact/k6"))
-              .withNetwork(NETWORK)
-              .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app")
-              .withCommand("run", "-u", "5", "-i", "200", "/app/basic.js")
-              .withStartupCheckStrategy(new OneShotStartupCheckStrategy());
-      k6.start();
+  private void doWarmupPhase(int seconds) {
+    System.out.println("Performing warm up phase for " + seconds + " seconds.");
+    GenericContainer<?> k6 =
+        new GenericContainer<>(DockerImageName.parse("loadimpact/k6"))
+            .withNetwork(NETWORK)
+            .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app")
+            .withCommand("run", "-u", "5", "-d", seconds + "s", "/app/performanceTest.js")
+            .withStartupCheckStrategy(
+                new OneShotStartupCheckStrategy().withTimeout(Duration.ofSeconds(seconds * 2L)));
+    k6.start();
+    sleep(seconds);
+    System.out.println("Awaiting warmup phase end.");
+    while (k6.isRunning()) {
+      System.out.println("Warmup still running.");
+      sleep(1);
     }
-
-    System.out.println("Stopping disposable JFR warmup recording...");
-    String[] stopCommand = {"jcmd", "1", "JFR.stop", "name=warmup"};
-    vehicleInventoryService.execInContainer(stopCommand);
-
     System.out.println("Warmup complete.");
   }
 
@@ -194,4 +177,12 @@ private void writeStartupTimeFile(DistroConfig distroConfig, long start) throws
     Path startupPath = namingConventions.local.startupDurationFile(distroConfig);
     Files.writeString(startupPath, String.valueOf(delta));
   }
+
+  private static void sleep(int seconds) {
+    try {
+      Thread.sleep(seconds * 1000L);
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
+    }
+  }
 }

performance-tests/src/test/java/io/opentelemetry/config/Configs.java

@@ -17,18 +17,20 @@ public enum Configs {
           .name("all-100-tps")
           .description("Compares all DistroConfigs (100TPS test)")
           .withDistroConfigs(DistroConfig.values())
-          .warmupSeconds(60)
+          .warmupSeconds(10)
           .maxRequestRate(100)
           .duration(System.getenv("DURATION"))
+          .concurrentConnections(System.getenv("CONCURRENCY"))
           .build()),
   ALL_800_TPS(
       TestConfig.builder()
           .name("all-800-tps")
           .description("Compares all DistroConfigs (800TPS test)")
           .withDistroConfigs(DistroConfig.values())
-          .warmupSeconds(60)
+          .warmupSeconds(10)
           .maxRequestRate(800)
           .duration(System.getenv("DURATION"))
+          .concurrentConnections(System.getenv("CONCURRENCY"))
           .build());
 
   public final TestConfig config;

performance-tests/src/test/java/io/opentelemetry/config/TestConfig.java

@@ -98,8 +98,10 @@ Builder maxRequestRate(int maxRequestRate) {
       return this;
     }
 
-    Builder concurrentConnections(int concurrentConnections) {
-      this.concurrentConnections = concurrentConnections;
+    Builder concurrentConnections(String concurrentConnections) {
+      if (concurrentConnections != null && !concurrentConnections.isEmpty()) {
+        this.concurrentConnections = Integer.parseInt(concurrentConnections);
+      }
       return this;
     }