Skip to content

Commit 1e72c0c

Browse files
vaindvaind
authored
ci(replay): Overhead measurement (#6611)
Add overhead measurements to measure and compare the performance impact of the Sentry Browser SDK and Replay: - Add three measurement scenarios: - Plain app without Sentry - App with Sentry+Tracing - App with Sentry+Tracing+Replay - Measure heap memory, CPU and web vitals impact - Add GHA job to add comment with results if a label is added to a PR
1 parent ed136de commit 1e72c0c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2444
-2
lines changed

‎.github/workflows/build.yml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -815,3 +815,51 @@ jobs:
815815
if: contains(needs.*.result, 'failure')
816816
run: |
817817
echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1
818+
819+
replay_metrics:
820+
name: Replay Metrics
821+
needs: [job_get_metadata, job_build]
822+
runs-on: ubuntu-20.04
823+
timeout-minutes: 30
824+
if: contains(github.event.pull_request.labels.*.name, 'ci-overhead-measurements')
825+
steps:
826+
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
827+
uses: actions/checkout@v3
828+
with:
829+
ref: ${{ env.HEAD_COMMIT }}
830+
- name: Set up Node
831+
uses: volta-cli/action@v4
832+
- name: Check dependency cache
833+
uses: actions/cache@v3
834+
with:
835+
path: ${{ env.CACHED_DEPENDENCY_PATHS }}
836+
key: ${{ needs.job_build.outputs.dependency_cache_key }}
837+
- name: Check build cache
838+
uses: actions/cache@v3
839+
with:
840+
path: ${{ env.CACHED_BUILD_PATHS }}
841+
key: ${{ env.BUILD_CACHE_KEY }}
842+
843+
- name: Setup
844+
run: yarn install
845+
working-directory: packages/replay/metrics
846+
847+
- name: Collect
848+
run: yarn ci:collect
849+
working-directory: packages/replay/metrics
850+
851+
- name: Process
852+
id: process
853+
run: yarn ci:process
854+
working-directory: packages/replay/metrics
855+
# Don't run on forks - the PR comment cannot be added.
856+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
857+
env:
858+
GITHUB_TOKEN: ${{ github.token }}
859+
860+
- name: Upload results
861+
uses: actions/upload-artifact@v3
862+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
863+
with:
864+
name: ${{ steps.process.outputs.artifactName }}
865+
path: ${{ steps.process.outputs.artifactPath }}

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,4 @@ tmp.js
4747

4848
# eslint
4949
.eslintcache
50-
eslintcache/*
50+
**/eslintcache/*

‎.vscode/launch.json

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,22 @@
3737
"internalConsoleOptions": "openOnSessionStart",
3838
"outputCapture": "std"
3939
},
40+
{
41+
"type": "node",
42+
"name": "Debug replay metrics collection script",
43+
"request": "launch",
44+
"cwd": "${workspaceFolder}/packages/replay/metrics/",
45+
"program": "${workspaceFolder}/packages/replay/metrics/configs/dev/collect.ts",
46+
"preLaunchTask": "Build Replay metrics script",
47+
},
48+
{
49+
"type": "node",
50+
"name": "Debug replay metrics processing script",
51+
"request": "launch",
52+
"cwd": "${workspaceFolder}/packages/replay/metrics/",
53+
"program": "${workspaceFolder}/packages/replay/metrics/configs/dev/process.ts",
54+
"preLaunchTask": "Build Replay metrics script",
55+
},
4056
// Run rollup using the config file which is in the currently active tab.
4157
{
4258
"name": "Debug rollup (config from open file)",

‎.vscode/tasks.json

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,13 @@
77
"type": "npm",
88
"script": "predebug",
99
"path": "packages/nextjs/test/integration/",
10-
"detail": "Link the SDK (if not already linked) and build test app"
10+
"detail": "Link the SDK (if not already linked) and build test app",
11+
},
12+
{
13+
"label": "Build Replay metrics script",
14+
"type": "npm",
15+
"script": "build",
16+
"path": "packages/replay/metrics",
1117
}
1218
]
1319
}

‎packages/replay/.eslintignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ build/
33
demo/build/
44
# TODO: Check if we can re-introduce linting in demo
55
demo
6+
metrics

‎packages/replay/metrics/.eslintrc.cjs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
module.exports = {
2+
extends: ['../.eslintrc.js'],
3+
ignorePatterns: ['test-apps'],
4+
overrides: [
5+
{
6+
files: ['*.ts'],
7+
rules: {
8+
'no-console': 'off',
9+
'@typescript-eslint/no-non-null-assertion': 'off',
10+
'import/no-unresolved': 'off',
11+
},
12+
},
13+
],
14+
};

‎packages/replay/metrics/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
out

‎packages/replay/metrics/README.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Replay performance metrics
2+
3+
Evaluates Replay impact on website performance by running a web app in Chromium via Playwright and collecting various metrics.
4+
5+
The general idea is to run a web app without Sentry Replay and then run the same app again with Sentry and another one with Sentry+Replay included.
6+
For the three scenarios, we collect some metrics (CPU, memory, vitals) and later compare them and post as a comment in a PR.
7+
Changes in the metrics, compared to previous runs from the main branch, should be evaluated on case-by-case basis when preparing and reviewing the PR.
8+
9+
## Resources
10+
11+
* https://github.com/addyosmani/puppeteer-webperf

‎packages/replay/metrics/configs/README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Replay metrics configuration & entrypoints (scripts)
2+
3+
* [dev](dev) contains scripts launched during local development
4+
* [ci](ci) contains scripts launched in CI

‎packages/replay/metrics/configs/ci/collect.ts

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import { Metrics, MetricsCollector } from '../../src/collector.js';
2+
import { MetricsStats, NumberProvider } from '../../src/results/metrics-stats.js';
3+
import { JankTestScenario } from '../../src/scenarios.js';
4+
import { printStats } from '../../src/util/console.js';
5+
import { latestResultFile } from './env.js';
6+
7+
function checkStdDev(results: Metrics[], name: string, provider: NumberProvider, max: number): boolean {
8+
const value = MetricsStats.stddev(results, provider);
9+
if (value == undefined) {
10+
console.warn(`✗ | Discarding results because StandardDeviation(${name}) is undefined`);
11+
return false;
12+
} else if (value > max) {
13+
console.warn(`✗ | Discarding results because StandardDeviation(${name}) is larger than ${max}. Actual value: ${value}`);
14+
return false;
15+
} else {
16+
console.log(`✓ | StandardDeviation(${name}) is ${value} (<= ${max})`)
17+
}
18+
return true;
19+
}
20+
21+
const collector = new MetricsCollector({ headless: true, cpuThrottling: 2 });
22+
const result = await collector.execute({
23+
name: 'jank',
24+
scenarios: [
25+
new JankTestScenario('index.html'),
26+
new JankTestScenario('with-sentry.html'),
27+
new JankTestScenario('with-replay.html'),
28+
],
29+
runs: 10,
30+
tries: 10,
31+
async shouldAccept(results: Metrics[]): Promise<boolean> {
32+
await printStats(results);
33+
34+
if (!checkStdDev(results, 'lcp', MetricsStats.lcp, 50)
35+
|| !checkStdDev(results, 'cls', MetricsStats.cls, 0.1)
36+
|| !checkStdDev(results, 'cpu', MetricsStats.cpu, 1)
37+
|| !checkStdDev(results, 'memory-mean', MetricsStats.memoryMean, 1000 * 1024)
38+
|| !checkStdDev(results, 'memory-max', MetricsStats.memoryMax, 1000 * 1024)) {
39+
return false;
40+
}
41+
42+
const cpuUsage = MetricsStats.mean(results, MetricsStats.cpu)!;
43+
if (cpuUsage > 0.85) {
44+
// Note: complexity on the "JankTest" is defined by the `minimum = ...,` setting in app.js - specifying the number of animated elements.
45+
console.warn(`✗ | Discarding results because CPU usage is too high and may be inaccurate: ${(cpuUsage * 100).toFixed(2)} %.`,
46+
'Consider simplifying the scenario or changing the CPU throttling factor.');
47+
return false;
48+
}
49+
50+
return true;
51+
},
52+
});
53+
54+
result.writeToFile(latestResultFile);

‎packages/replay/metrics/configs/ci/env.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
export const previousResultsDir = 'out/previous-results';
2+
export const baselineResultsDir = 'out/baseline-results';
3+
export const latestResultFile = 'out/latest-result.json';
4+
export const artifactName = 'replay-sdk-metrics'

‎packages/replay/metrics/configs/ci/process.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import path from 'path';
2+
3+
import { ResultsAnalyzer } from '../../src/results/analyzer.js';
4+
import { PrCommentBuilder } from '../../src/results/pr-comment.js';
5+
import { Result } from '../../src/results/result.js';
6+
import { ResultsSet } from '../../src/results/results-set.js';
7+
import { Git } from '../../src/util/git.js';
8+
import { GitHub } from '../../src/util/github.js';
9+
import { artifactName, baselineResultsDir, latestResultFile, previousResultsDir } from './env.js';
10+
11+
const latestResult = Result.readFromFile(latestResultFile);
12+
const branch = await Git.branch;
13+
const baseBranch = await Git.baseBranch;
14+
15+
await GitHub.downloadPreviousArtifact(baseBranch, baselineResultsDir, artifactName);
16+
await GitHub.downloadPreviousArtifact(branch, previousResultsDir, artifactName);
17+
18+
GitHub.writeOutput('artifactName', artifactName)
19+
GitHub.writeOutput('artifactPath', path.resolve(previousResultsDir));
20+
21+
const previousResults = new ResultsSet(previousResultsDir);
22+
23+
const prComment = new PrCommentBuilder();
24+
if (baseBranch != branch) {
25+
const baseResults = new ResultsSet(baselineResultsDir);
26+
await prComment.addCurrentResult(await ResultsAnalyzer.analyze(latestResult, baseResults), 'Baseline');
27+
await prComment.addAdditionalResultsSet(
28+
`Baseline results on branch: <code>${baseBranch}</code>`,
29+
// We skip the first one here because it's already included as `Baseline` column above in addCurrentResult().
30+
baseResults.items().slice(1, 10)
31+
);
32+
} else {
33+
await prComment.addCurrentResult(await ResultsAnalyzer.analyze(latestResult, previousResults), 'Previous');
34+
}
35+
36+
await prComment.addAdditionalResultsSet(
37+
`Previous results on branch: <code>${branch}</code>`,
38+
previousResults.items().slice(0, 10)
39+
);
40+
41+
await GitHub.addOrUpdateComment(prComment);
42+
43+
// Copy the latest test run results to the archived result dir.
44+
await previousResults.add(latestResultFile, true);

‎packages/replay/metrics/configs/dev/collect.ts

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { Metrics, MetricsCollector } from '../../src/collector.js';
2+
import { MetricsStats } from '../../src/results/metrics-stats.js';
3+
import { JankTestScenario } from '../../src/scenarios.js';
4+
import { printStats } from '../../src/util/console.js';
5+
import { latestResultFile } from './env.js';
6+
7+
const collector = new MetricsCollector();
8+
const result = await collector.execute({
9+
name: 'dummy',
10+
scenarios: [
11+
new JankTestScenario('index.html'),
12+
new JankTestScenario('with-sentry.html'),
13+
new JankTestScenario('with-replay.html'),
14+
],
15+
runs: 1,
16+
tries: 1,
17+
async shouldAccept(results: Metrics[]): Promise<boolean> {
18+
printStats(results);
19+
20+
const cpuUsage = MetricsStats.mean(results, MetricsStats.cpu)!;
21+
if (cpuUsage > 0.9) {
22+
console.error(`CPU usage too high to be accurate: ${(cpuUsage * 100).toFixed(2)} %.`,
23+
'Consider simplifying the scenario or changing the CPU throttling factor.');
24+
return false;
25+
}
26+
return true;
27+
},
28+
});
29+
30+
result.writeToFile(latestResultFile);

‎packages/replay/metrics/configs/dev/env.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export const outDir = 'out/results-dev';
2+
export const latestResultFile = 'out/latest-result.json';

‎packages/replay/metrics/configs/dev/process.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { ResultsAnalyzer } from '../../src/results/analyzer.js';
2+
import { Result } from '../../src/results/result.js';
3+
import { ResultsSet } from '../../src/results/results-set.js';
4+
import { printAnalysis } from '../../src/util/console.js';
5+
import { latestResultFile, outDir } from './env.js';
6+
7+
const resultsSet = new ResultsSet(outDir);
8+
const latestResult = Result.readFromFile(latestResultFile);
9+
10+
const analysis = await ResultsAnalyzer.analyze(latestResult, resultsSet);
11+
printAnalysis(analysis);
12+
13+
await resultsSet.add(latestResultFile, true);

‎packages/replay/metrics/package.json

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"private": true,
3+
"name": "metrics",
4+
"main": "index.js",
5+
"author": "Sentry",
6+
"license": "MIT",
7+
"type": "module",
8+
"scripts": {
9+
"build": "tsc",
10+
"deps": "yarn --cwd ../ build:bundle && yarn --cwd ../../tracing/ build:bundle",
11+
"dev:collect": "ts-node-esm ./configs/dev/collect.ts",
12+
"dev:process": "ts-node-esm ./configs/dev/process.ts",
13+
"ci:collect": "ts-node-esm ./configs/ci/collect.ts",
14+
"ci:process": "ts-node-esm ./configs/ci/process.ts"
15+
},
16+
"dependencies": {
17+
"@octokit/rest": "^19.0.5",
18+
"@types/node": "^18.11.17",
19+
"axios": "^1.2.2",
20+
"extract-zip": "^2.0.1",
21+
"filesize": "^10.0.6",
22+
"p-timeout": "^6.0.0",
23+
"playwright": "^1.29.1",
24+
"playwright-core": "^1.29.1",
25+
"simple-git": "^3.15.1",
26+
"simple-statistics": "^7.8.0",
27+
"typescript": "^4.9.4"
28+
},
29+
"devDependencies": {
30+
"ts-node": "^10.9.1"
31+
}
32+
}

0 commit comments

Comments
 (0)