Skip to content

Commit bdd81df

Browse files
AsakerMohdAsakerMohd
authored
Added new python profiler for performance testing (#89)
*Description of changes:* Added a new profiler.py script that uses psutils to get per process info such as Network Bytes Sent/Received, CPU, Resident Memory (Phyical RAM used), virtual memory and the peak number of threads. Updated the persisters as well to read the new data and populate `summary.txt` and `results.csv` as expected. Below is an example from a test run. Also added a bunch of TODO comments to clean up old JFR logic that is no longer required here and to keep this PR changes to a minimum. ``` ---------------------------------------------------------- Run at Mon Mar 04 15:53:15 PST 2024 all-100-tps : Compares all DistroConfigs (100TPS test) 5 users, 10s duration ---------------------------------------------------------- DistroConfig : none app_signals_disabled app_signals_no_traces app_signals_traces Run duration : 00:00:19 00:00:15 00:00:21 00:00:17 Startup time (ms) : 4040 5051 5042 5058 Req. Count : 504.00 504.00 462.00 448.00 Req. Rate : 31.97 44.39 25.53 33.31 Req. Lat. mean (ms) : 113.25 106.52 130.56 121.27 Req. Lat. p0 (ms) : 1.56 2.11 2.09 2.07 Req. Lat. p50 (ms) : 54.95 56.82 56.22 56.85 Req. Lat. p90 (ms) : 336.17 340.21 353.73 350.77 Req. Lat. p99 (ms) : 420.83 429.16 441.63 458.23 Req. Lat. p100 (ms) : 5410.95 5436.79 10113.78 5405.26 Net Sent mean (B) : 47226.00 64212.00 39314.00 49675.00 Net Sent p0 (B) : 0.00 0.00 0.00 0.00 Net Sent p50 (B) : 62080.00 72615.00 29153.00 55584.00 Net Sent p90 (B) : 97788.00 104820.00 104455.00 87291.00 Net Sent p99 (B) : 107597.00 128103.00 112795.00 111525.00 Net Sent p100 (B) : 107597.00 128103.00 112795.00 111525.00 Net Recv mean (B) : 46004.00 58401.00 38055.00 45899.00 Net Recv p0 (B) : 0.00 0.00 0.00 0.00 Net Recv p50 (B) : 60268.00 70027.00 29353.00 55344.00 Net Recv p90 (B) : 94835.00 94095.00 102147.00 81180.00 Net Recv p99 (B) : 105284.00 101951.00 108862.00 107865.00 Net Recv p100 (B) : 105284.00 101951.00 108862.00 107865.00 CPU Usage mean % : 1.73 2.71 1.81 2.17 CPU Usage p0 % : 0.00 0.00 0.00 0.00 CPU Usage p50 % : 2.07 2.99 1.24 2.49 CPU Usage p90 % : 3.73 4.56 4.81 4.48 CPU Usage p99 % : 4.23 4.73 5.22 5.30 CPU Usage p100 % : 4.23 4.73 5.22 5.30 RSS Mem mean (MB) : 50.65 81.07 83.21 80.96 RSS Mem p0 (MB) : 49.78 78.30 79.71 79.33 RSS Mem p50 (MB) : 50.78 81.30 83.53 81.21 RSS Mem p90 (MB) : 50.91 82.17 83.53 81.58 RSS Mem p99 (MB) : 50.91 82.17 87.15 81.58 RSS Mem p100 (MB) : 50.91 82.17 87.15 81.58 VMS Mem mean (MB) : 445.26 2100.96 1928.66 2114.42 VMS Mem p0 (MB) : 203.14 1746.01 1601.97 1746.04 VMS Mem p50 (MB) : 491.16 2179.04 1962.99 2179.07 VMS Mem p90 (MB) : 491.16 2179.04 1962.99 2179.07 VMS Mem p99 (MB) : 491.16 2179.04 1964.99 2179.07 VMS Mem p100 (MB) : 491.16 2179.04 1964.99 2179.07 Peak threads : 7 29 27 29 ``` ### Testing ``` asakem@88665a24d661 performance-tests % ./gradlew test Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0. You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins. For more on this, please refer to https://docs.gradle.org/8.6/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation. BUILD SUCCESSFUL in 15m 14s 3 actionable tasks: 2 executed, 1 up-to-date ``` By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
1 parent b81f2f9 commit bdd81df

File tree

11 files changed

+515
-52
lines changed

11 files changed

+515
-52
lines changed

‎performance-tests/Dockerfile-VehicleInventoryService-base

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,5 @@ COPY ./sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp /ve
2020
COPY ./dist/$DISTRO /vehicle-inventory-app/
2121

2222
# Install dependencies and distro
23-
RUN pip install --upgrade pip && pip install -r requirements.txt && pip install ${DISTRO} --force-reinstall
23+
RUN pip install --upgrade pip && pip install -r requirements.txt && pip install psutil && pip install ${DISTRO} \
24+
--force-reinstall

‎performance-tests/src/test/java/io/opentelemetry/OverheadTests.java

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
101101
imageService.start();
102102

103103
GenericContainer<?> vehicleInventoryService =
104-
new VehicleInventoryServiceContainer(NETWORK, collector, distroConfig).build();
104+
new VehicleInventoryServiceContainer(NETWORK, collector, distroConfig, namingConventions)
105+
.build();
105106
long start = System.currentTimeMillis();
106107
vehicleInventoryService.start();
107108
writeStartupTimeFile(distroConfig, start);
@@ -114,7 +115,7 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
114115
}
115116

116117
long testStart = System.currentTimeMillis();
117-
// startRecording(distroConfig, vehicleInventoryService);
118+
startRecording(distroConfig, vehicleInventoryService);
118119

119120
GenericContainer<?> k6 =
120121
new K6Container(NETWORK, distroConfig, config, namingConventions).build();
@@ -130,15 +131,11 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
130131

131132
private void startRecording(
132133
DistroConfig distroConfig, GenericContainer<?> vehicleInventoryService) throws Exception {
133-
Path outFile = namingConventions.container.jfrFile(distroConfig);
134134
String[] command = {
135-
"jcmd",
136-
"1",
137-
"JFR.start",
138-
"settings=/app/overhead.jfc",
139-
"dumponexit=true",
140-
"name=petclinic",
141-
"filename=" + outFile
135+
"sh",
136+
"executeProfiler.sh",
137+
namingConventions.container.performanceMetricsFileWithoutPath(distroConfig),
138+
namingConventions.container.root()
142139
};
143140
vehicleInventoryService.execInContainer(command);
144141
}

‎performance-tests/src/test/java/io/opentelemetry/containers/VehicleInventoryServiceContainer.java

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
package io.opentelemetry.containers;
88

99
import io.opentelemetry.distros.DistroConfig;
10+
import io.opentelemetry.util.NamingConventions;
1011
import io.opentelemetry.util.RuntimeUtil;
1112
import org.slf4j.Logger;
1213
import org.slf4j.LoggerFactory;
@@ -28,12 +29,17 @@ public class VehicleInventoryServiceContainer {
2829
private final Network network;
2930
private final Startable collector;
3031
private final DistroConfig distroConfig;
32+
private final NamingConventions namingConventions;
3133

3234
public VehicleInventoryServiceContainer(
33-
Network network, Startable collector, DistroConfig distroConfig) {
35+
Network network,
36+
Startable collector,
37+
DistroConfig distroConfig,
38+
NamingConventions namingConventions) {
3439
this.network = network;
3540
this.collector = collector;
3641
this.distroConfig = distroConfig;
42+
this.namingConventions = namingConventions;
3743
}
3844

3945
public GenericContainer<?> build() {
@@ -44,9 +50,17 @@ public GenericContainer<?> build() {
4450
.withLogConsumer(new Slf4jLogConsumer(logger))
4551
.withExposedPorts(PORT)
4652
.waitingFor(Wait.forHttp("/vehicle-inventory/health-check").forPort(PORT))
53+
.withFileSystemBind(
54+
namingConventions.localResults(), namingConventions.containerResults())
4755
.withCopyFileToContainer(
4856
MountableFile.forClasspathResource("runVehicleInventory.sh"),
4957
"vehicle-inventory-app/run.sh")
58+
.withCopyFileToContainer(
59+
MountableFile.forClasspathResource("profiler.py"),
60+
"vehicle-inventory-app/profiler.py")
61+
.withCopyFileToContainer(
62+
MountableFile.forClasspathResource("executeProfiler.sh"),
63+
"vehicle-inventory-app/executeProfiler.sh")
5064
.withEnv("DJANGO_SETTINGS_MODULE", "VehicleInventoryApp.settings")
5165
.withEnv("PORT", Integer.toString(PORT))
5266
.withEnv("POSTGRES_DATABASE", PostgresContainer.DATABASE_NAME)

‎performance-tests/src/test/java/io/opentelemetry/results/AppPerfResults.java

Lines changed: 102 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,49 @@ public class AppPerfResults {
2121
final double requestLatencyP90;
2222
final double requestLatencyP99;
2323
final double requestLatencyP100;
24+
final long networkBytesSentAvg;
25+
final long networkBytesSentP0;
26+
final long networkBytesSentP50;
27+
final long networkBytesSentP90;
28+
final long networkBytesSentP99;
29+
final long networkBytesSentP100;
30+
final long networkBytesRecvAvg;
31+
final long networkBytesRecvP0;
32+
final long networkBytesRecvP50;
33+
final long networkBytesRecvP90;
34+
final long networkBytesRecvP99;
35+
final long networkBytesRecvP100;
36+
final double cpuAvg;
37+
final double cpuP0;
38+
final double cpuP50;
39+
final double cpuP90;
40+
final double cpuP99;
41+
final double cpuP100;
42+
final long rssMemAvg;
43+
final long rssMemP0;
44+
final long rssMemP50;
45+
final long rssMemP90;
46+
final long rssMemP99;
47+
final long rssMemP100;
48+
final long vmsMemAvg;
49+
final long vmsMemP0;
50+
final long vmsMemP50;
51+
final long vmsMemP90;
52+
final long vmsMemP99;
53+
final long vmsMemP100;
54+
final long peakThreadCount;
55+
final long startupDurationMs;
56+
final long runDurationMs;
57+
// TODO: cleanup
2458
final long totalGCTime;
2559
final long totalAllocated;
2660
final MinMax heapUsed;
2761
final float maxThreadContextSwitchRate;
28-
final long startupDurationMs;
29-
final long peakThreadCount;
3062
final long averageNetworkRead;
3163
final long averageNetworkWrite;
3264
final float averageJvmUserCpu;
3365
final float maxJvmUserCpu;
3466
final float averageMachineCpuTotal;
35-
final long runDurationMs;
3667
final long totalGcPauseNanos;
3768

3869
private AppPerfResults(Builder builder) {
@@ -46,18 +77,48 @@ private AppPerfResults(Builder builder) {
4677
this.requestLatencyP90 = builder.requestLatencyP90;
4778
this.requestLatencyP99 = builder.requestLatencyP99;
4879
this.requestLatencyP100 = builder.requestLatencyP100;
80+
this.networkBytesSentAvg = builder.networkBytesSentAvg;
81+
this.networkBytesSentP0 = builder.networkBytesSentP0;
82+
this.networkBytesSentP50 = builder.networkBytesSentP50;
83+
this.networkBytesSentP90 = builder.networkBytesSentP90;
84+
this.networkBytesSentP99 = builder.networkBytesSentP99;
85+
this.networkBytesSentP100 = builder.networkBytesSentP100;
86+
this.networkBytesRecvAvg = builder.networkBytesRecvAvg;
87+
this.networkBytesRecvP0 = builder.networkBytesRecvP0;
88+
this.networkBytesRecvP50 = builder.networkBytesRecvP50;
89+
this.networkBytesRecvP90 = builder.networkBytesRecvP90;
90+
this.networkBytesRecvP99 = builder.networkBytesRecvP99;
91+
this.networkBytesRecvP100 = builder.networkBytesRecvP100;
92+
this.cpuAvg = builder.cpuAvg;
93+
this.cpuP0 = builder.cpuP0;
94+
this.cpuP50 = builder.cpuP50;
95+
this.cpuP90 = builder.cpuP90;
96+
this.cpuP99 = builder.cpuP99;
97+
this.cpuP100 = builder.cpuP100;
98+
this.rssMemAvg = builder.rssMemAvg;
99+
this.rssMemP0 = builder.rssMemP0;
100+
this.rssMemP50 = builder.rssMemP50;
101+
this.rssMemP90 = builder.rssMemP90;
102+
this.rssMemP99 = builder.rssMemP99;
103+
this.rssMemP100 = builder.rssMemP100;
104+
this.vmsMemAvg = builder.vmsMemAvg;
105+
this.vmsMemP0 = builder.vmsMemP0;
106+
this.vmsMemP50 = builder.vmsMemP50;
107+
this.vmsMemP90 = builder.vmsMemP90;
108+
this.vmsMemP99 = builder.vmsMemP99;
109+
this.vmsMemP100 = builder.vmsMemP100;
110+
this.peakThreadCount = builder.peakThreadCount;
111+
this.startupDurationMs = builder.startupDurationMs;
112+
this.runDurationMs = builder.runDurationMs;
49113
this.totalGCTime = builder.totalGCTime;
50114
this.totalAllocated = builder.totalAllocated;
51115
this.heapUsed = builder.heapUsed;
52116
this.maxThreadContextSwitchRate = builder.maxThreadContextSwitchRate;
53-
this.startupDurationMs = builder.startupDurationMs;
54-
this.peakThreadCount = builder.peakThreadCount;
55117
this.averageNetworkRead = builder.averageNetworkRead;
56118
this.averageNetworkWrite = builder.averageNetworkWrite;
57119
this.averageJvmUserCpu = builder.averageJvmUserCpu;
58120
this.maxJvmUserCpu = builder.maxJvmUserCpu;
59121
this.averageMachineCpuTotal = builder.averageMachineCpuTotal;
60-
this.runDurationMs = builder.runDurationMs;
61122
this.totalGcPauseNanos = builder.totalGcPauseNanos;
62123
}
63124

@@ -73,7 +134,7 @@ private AppPerfResults(Builder builder) {
73134
return bytesToMegs(this.heapUsed.max);
74135
}
75136

76-
private double bytesToMegs(long x) {
137+
public double bytesToMegs(long x) {
77138
return x / (1024.0 * 1024.0);
78139
}
79140

@@ -97,17 +158,48 @@ static class Builder {
97158
public double requestLatencyP90;
98159
public double requestLatencyP99;
99160
public double requestLatencyP100;
161+
public long networkBytesSentAvg;
162+
public long networkBytesSentP0;
163+
public long networkBytesSentP50;
164+
public long networkBytesSentP90;
165+
public long networkBytesSentP99;
166+
public long networkBytesSentP100;
167+
public long networkBytesRecvAvg;
168+
public long networkBytesRecvP0;
169+
public long networkBytesRecvP50;
170+
public long networkBytesRecvP90;
171+
public long networkBytesRecvP99;
172+
public long networkBytesRecvP100;
173+
public double cpuAvg;
174+
public double cpuP0;
175+
public double cpuP50;
176+
public double cpuP90;
177+
public double cpuP99;
178+
public double cpuP100;
179+
public long rssMemAvg;
180+
public long rssMemP0;
181+
public long rssMemP50;
182+
public long rssMemP90;
183+
public long rssMemP99;
184+
public long rssMemP100;
185+
public long vmsMemAvg;
186+
public long vmsMemP0;
187+
public long vmsMemP50;
188+
public long vmsMemP90;
189+
public long vmsMemP99;
190+
public long vmsMemP100;
191+
public long peakThreadCount;
192+
public long runDurationMs;
193+
// TODO: cleanup
100194
private long totalGCTime;
101195
private long totalAllocated;
102196
private MinMax heapUsed;
103197
private float maxThreadContextSwitchRate;
104-
private long peakThreadCount;
105198
public long averageNetworkRead;
106199
public long averageNetworkWrite;
107200
public float averageJvmUserCpu;
108201
public float maxJvmUserCpu;
109202
public float averageMachineCpuTotal;
110-
public long runDurationMs;
111203
public long totalGcPauseNanos;
112204

113205
AppPerfResults build() {
@@ -124,6 +216,7 @@ Builder config(TestConfig config) {
124216
return this;
125217
}
126218

219+
// TODO: cleanup
127220
Builder totalGCTime(long totalGCTime) {
128221
this.totalGCTime = totalGCTime;
129222
return this;

‎performance-tests/src/test/java/io/opentelemetry/results/CsvPersister.java

Lines changed: 37 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@
66

77
package io.opentelemetry.results;
88

9-
import static java.util.concurrent.TimeUnit.NANOSECONDS;
10-
119
import java.io.IOException;
1210
import java.nio.file.Files;
1311
import java.nio.file.Path;
@@ -23,11 +21,6 @@ class CsvPersister implements ResultsPersister {
2321
private static final List<FieldSpec> FIELDS =
2422
Arrays.asList(
2523
FieldSpec.of("startupDurationMs", r -> r.startupDurationMs),
26-
// FieldSpec.of("minHeapUsed", r -> r.heapUsed.min),
27-
// FieldSpec.of("maxHeapUsed", r -> r.heapUsed.max),
28-
FieldSpec.of("totalAllocatedMB", r -> r.getTotalAllocatedMB()),
29-
FieldSpec.of("totalGCTime", r -> r.totalGCTime),
30-
FieldSpec.of("maxThreadContextSwitchRate", r -> r.maxThreadContextSwitchRate),
3124
FieldSpec.of("requestCount", r -> r.requestCount),
3225
FieldSpec.of("requestRate", r -> r.requestRate),
3326
FieldSpec.of("requestLatencyAvg", r -> r.requestLatencyAvg),
@@ -36,14 +29,44 @@ class CsvPersister implements ResultsPersister {
3629
FieldSpec.of("requestLatencyP90", r -> r.requestLatencyP90),
3730
FieldSpec.of("requestLatencyP99", r -> r.requestLatencyP99),
3831
FieldSpec.of("requestLatencyP100", r -> r.requestLatencyP100),
39-
FieldSpec.of("netReadAvg", r -> r.averageNetworkRead),
40-
FieldSpec.of("netWriteAvg", r -> r.averageNetworkWrite),
32+
FieldSpec.of("networkBytesSentAvg", r -> r.networkBytesSentAvg),
33+
FieldSpec.of("networkBytesSentP0", r -> r.networkBytesSentP0),
34+
FieldSpec.of("networkBytesSentP50", r -> r.networkBytesSentP50),
35+
FieldSpec.of("networkBytesSentP90", r -> r.networkBytesSentP90),
36+
FieldSpec.of("networkBytesSentP99", r -> r.networkBytesSentP99),
37+
FieldSpec.of("networkBytesSentP100", r -> r.networkBytesSentP100),
38+
FieldSpec.of("networkBytesRecvAvg", r -> r.networkBytesRecvAvg),
39+
FieldSpec.of("networkBytesRecvP0", r -> r.networkBytesRecvP0),
40+
FieldSpec.of("networkBytesRecvP50", r -> r.networkBytesRecvP50),
41+
FieldSpec.of("networkBytesRecvP90", r -> r.networkBytesRecvP90),
42+
FieldSpec.of("networkBytesRecvP99", r -> r.networkBytesRecvP99),
43+
FieldSpec.of("networkBytesRecvP100", r -> r.networkBytesRecvP100),
44+
FieldSpec.of("cpuAvg", r -> r.cpuAvg),
45+
FieldSpec.of("cpuP0", r -> r.cpuP0),
46+
FieldSpec.of("cpuP50", r -> r.cpuP50),
47+
FieldSpec.of("cpuP90", r -> r.cpuP90),
48+
FieldSpec.of("cpuP99", r -> r.cpuP99),
49+
FieldSpec.of("cpuP100", r -> r.cpuP100),
50+
FieldSpec.of("cpuAvg", r -> r.cpuAvg),
51+
FieldSpec.of("cpuP0", r -> r.cpuP0),
52+
FieldSpec.of("cpuP50", r -> r.cpuP50),
53+
FieldSpec.of("cpuP90", r -> r.cpuP90),
54+
FieldSpec.of("cpuP99", r -> r.cpuP99),
55+
FieldSpec.of("cpuP100", r -> r.cpuP100),
56+
FieldSpec.of("rssMemAvg", r -> r.rssMemAvg),
57+
FieldSpec.of("rssMemP0", r -> r.rssMemP0),
58+
FieldSpec.of("rssMemP50", r -> r.rssMemP50),
59+
FieldSpec.of("rssMemP90", r -> r.rssMemP90),
60+
FieldSpec.of("rssMemP99", r -> r.rssMemP99),
61+
FieldSpec.of("rssMemP100", r -> r.rssMemP100),
62+
FieldSpec.of("vmsMemAvg", r -> r.vmsMemAvg),
63+
FieldSpec.of("vmsMemP0", r -> r.vmsMemP0),
64+
FieldSpec.of("vmsMemP50", r -> r.vmsMemP50),
65+
FieldSpec.of("vmsMemP90", r -> r.vmsMemP90),
66+
FieldSpec.of("vmsMemP99", r -> r.vmsMemP99),
67+
FieldSpec.of("vmsMemP100", r -> r.vmsMemP100),
4168
FieldSpec.of("peakThreadCount", r -> r.peakThreadCount),
42-
FieldSpec.of("averageCpuUser", r -> r.averageJvmUserCpu),
43-
FieldSpec.of("maxCpuUser", r -> r.maxJvmUserCpu),
44-
FieldSpec.of("averageMachineCpuTotal", r -> r.averageMachineCpuTotal),
45-
FieldSpec.of("runDurationMs", r -> r.runDurationMs),
46-
FieldSpec.of("gcPauseMs", r -> NANOSECONDS.toMillis(r.totalGcPauseNanos)));
69+
FieldSpec.of("runDurationMs", r -> r.runDurationMs));
4770

4871
private final Path resultsFile;
4972

0 commit comments

Comments
 (0)