Skip to content

Commit ebefe1f

Browse files
thetaPCthetaPC
authored
chore(visual-regression): add script to update ground truths (#29204)
Issue number: N/A --------- <!-- Please do not submit updates to dependencies unless it fixes an issue. --> <!-- Please try to limit your pull request to one type (bugfix, feature, etc). Submit multiple pull requests if needed. --> ## What is the current behavior? <!-- Please describe the current behavior that you are modifying. --> Devs would have to manually generate the ground truths from their desired base branch. This causes a dev to checkout the base branch and pull the latest screenshots. They would then return to their working branch and start the E2E tests. ## What is the new behavior? <!-- Please describe the behavior or changes that are being added by this PR. --> A script has been created to automate this process using Docker as mentioned in the design doc: - It will ask the user a set a questions like if which component they want to test ## Does this introduce a breaking change? - [ ] Yes - [x] No <!-- If this introduces a breaking change: 1. Describe the impact and migration path for existing applications below. 2. Update the BREAKING.md file with the breaking change. 3. Add "BREAKING CHANGE: [...]" to the commit description when merging. See https://github.com/ionic-team/ionic-framework/blob/main/.github/CONTRIBUTING.md#footer for more information. --> ## Other information <!-- Any other information that is important to this PR such as screenshots of how the component looks before and after the change. --> How to test: 1. Make a change to a desired component 2. Run `npm run test.e2e.script` 3. Answer the questions 4. Verify that the tests fail due to visual changes 5. Re-run the command as many times as necessary in order to try different routes based on different answers
1 parent 5cdfac8 commit ebefe1f

File tree

3 files changed

+330
-1
lines changed

3 files changed

+330
-1
lines changed

‎core/package-lock.json

Lines changed: 67 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎core/package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
"@capacitor/haptics": "^6.0.0",
4242
"@capacitor/keyboard": "^6.0.0",
4343
"@capacitor/status-bar": "^6.0.0",
44+
"@clack/prompts": "^0.7.0",
4445
"@ionic/eslint-config": "^0.3.0",
4546
"@ionic/prettier-config": "^2.0.0",
4647
"@playwright/test": "^1.39.0",
@@ -99,7 +100,8 @@
99100
"docker.build": "docker build -t ionic-playwright .",
100101
"test.e2e.docker": "npm run docker.build && node ./scripts/docker.mjs",
101102
"test.e2e.docker.update-snapshots": "npm run test.e2e.docker -- --update-snapshots",
102-
"test.e2e.docker.ci": "npm run docker.build && CI=true node ./scripts/docker.mjs"
103+
"test.e2e.docker.ci": "npm run docker.build && CI=true node ./scripts/docker.mjs",
104+
"test.e2e.script": "node scripts/testing/e2e-script.mjs"
103105
},
104106
"author": "Ionic Team",
105107
"license": "MIT",

‎core/scripts/testing/e2e-script.mjs

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
// The purpose of this script is to provide a way run the E2E tests
2+
// without having the developer to manually run multiple commands based
3+
// on the desired end result.
4+
// E.g. update the local ground truths for a specific component or
5+
// open the Playwright report after running the E2E tests.
6+
7+
import {
8+
intro,
9+
outro,
10+
confirm,
11+
spinner,
12+
isCancel,
13+
cancel,
14+
text,
15+
log,
16+
} from '@clack/prompts';
17+
import { exec, spawn } from 'child_process';
18+
import fs from 'node:fs';
19+
import { setTimeout as sleep } from 'node:timers/promises';
20+
import util from 'node:util';
21+
import color from 'picocolors';
22+
23+
async function main() {
24+
const execAsync = util.promisify(exec);
25+
const cleanUpFiles = async () => {
26+
// Clean up the local ground truths.
27+
const cleanUp = spinner();
28+
29+
// Inform the user that the local ground truths are being cleaned up.
30+
cleanUp.start('Restoring local ground truths');
31+
32+
// Reset the local ground truths.
33+
await execAsync('git reset -- src/**/*-linux.png').catch((error) => {
34+
cleanUp.stop('Failed to reset local ground truths');
35+
console.error(error);
36+
return process.exit(0);
37+
});
38+
39+
// Restore the local ground truths.
40+
await execAsync('git restore -- src/**/*-linux.png').catch((error) => {
41+
cleanUp.stop('Failed to restore local ground truths');
42+
console.error(error);
43+
return process.exit(0);
44+
});
45+
46+
// Inform the user that the local ground truths have been cleaned up.
47+
cleanUp.stop('Local ground truths have been restored to their original state in order to avoid committing them.');
48+
};
49+
50+
intro(color.inverse(' Update Local Ground Truths'));
51+
52+
// Ask user for the component name they want to test.
53+
const componentValue = await text({
54+
message: 'Enter the component or path you want to test (e.g. chip, src/components/chip)',
55+
placeholder: 'Empty for all components',
56+
});
57+
58+
// User cancelled the operation with `Ctrl+C` or `CMD+C`.
59+
if (isCancel(componentValue)) {
60+
cancel('Operation cancelled');
61+
return process.exit(0);
62+
}
63+
64+
// Ask user if they want to update their local ground truths.
65+
const shouldUpdateTruths = await confirm({
66+
message: 'Do you want to update your local ground truths?',
67+
});
68+
69+
// User cancelled the operation with `Ctrl+C` or `CMD+C`.
70+
if (isCancel(shouldUpdateTruths)) {
71+
cancel('Operation cancelled');
72+
return process.exit(0);
73+
}
74+
75+
if (shouldUpdateTruths) {
76+
const defaultBaseBranch = 'main';
77+
78+
// Ask user for the base branch.
79+
let baseBranch = await text({
80+
message: 'Enter the base branch name:',
81+
placeholder: `default: ${defaultBaseBranch}`,
82+
})
83+
84+
// User cancelled the operation with `Ctrl+C` or `CMD+C`.
85+
if (isCancel(baseBranch)) {
86+
cancel('Operation cancelled');
87+
return process.exit(0);
88+
}
89+
90+
// User didn't provide a base branch.
91+
if (!baseBranch) {
92+
baseBranch = defaultBaseBranch;
93+
}
94+
95+
/**
96+
* The provided base branch needs to be fetched.
97+
* This ensures that the local base branch is up-to-date with the
98+
* remote base branch. Otherwise, there might be errors stating that
99+
* certain files don't exist in the local base branch.
100+
*/
101+
const fetchBaseBranch = spinner();
102+
103+
// Inform the user that the base branch is being fetched.
104+
fetchBaseBranch.start(`Fetching "${baseBranch}" to have the latest changes`);
105+
106+
// Fetch the base branch.
107+
await execAsync(`git fetch origin ${baseBranch}`).catch((error) => {
108+
fetchBaseBranch.stop(`Failed to fetch "${baseBranch}"`);
109+
console.error(error);
110+
return process.exit(0);
111+
});
112+
113+
// Inform the user that the base branch has been fetched.
114+
fetchBaseBranch.stop(`Fetched "${baseBranch}"`);
115+
116+
117+
const updateGroundTruth = spinner();
118+
119+
// Inform the user that the local ground truths are being updated.
120+
updateGroundTruth.start('Updating local ground truths');
121+
122+
// Check if user provided an existing file or directory.
123+
const isValidLocation = fs.existsSync(componentValue);
124+
125+
// User provided an existing file or directory.
126+
if (isValidLocation) {
127+
const stats = fs.statSync(componentValue);
128+
129+
// User provided a file as the component.
130+
// ex: `componentValue` = `src/components/chip/test/basic/chip.e2e.ts`
131+
if (stats.isFile()) {
132+
// Update the local ground truths for the provided path.
133+
await execAsync(`git checkout origin/${baseBranch} -- ${componentValue}-snapshots/*-linux.png`).catch((error) => {
134+
updateGroundTruth.stop('Failed to update local ground truths');
135+
console.error(error);
136+
return process.exit(0);
137+
});
138+
}
139+
140+
// User provided a directory as the component.
141+
// ex: `componentValue` = `src/components/chip`
142+
if (stats.isDirectory()) {
143+
// Update the local ground truths for the provided directory.
144+
await execAsync(`git checkout origin/${baseBranch} -- ${componentValue}/test/*/*.e2e.ts-snapshots/*-linux.png`).catch((error) => {
145+
updateGroundTruth.stop('Failed to update local ground truths');
146+
console.error(error);
147+
return process.exit(0);
148+
});
149+
}
150+
}
151+
// User provided a component name as the component.
152+
// ex: `componentValue` = `chip`
153+
else if (componentValue) {
154+
// Update the local ground truths for the provided component.
155+
await execAsync(`git checkout origin/${baseBranch} -- src/components/${componentValue}/test/*/${componentValue}.e2e.ts-snapshots/*-linux.png`).catch((error) => {
156+
updateGroundTruth.stop('Failed to update local ground truths');
157+
console.error(error);
158+
return process.exit(0);
159+
});
160+
}
161+
// User provided an empty string.
162+
else {
163+
// Update the local ground truths for all components.
164+
await execAsync(`git checkout origin/${baseBranch} -- src/components/*/test/*/*.e2e.ts-snapshots/*-linux.png`).catch((error) => {
165+
updateGroundTruth.stop('Failed to update local ground truths');
166+
console.error(error);
167+
return process.exit(0);
168+
});
169+
}
170+
171+
// Inform the user that the local ground truths have been updated.
172+
updateGroundTruth.stop('Updated local ground truths');
173+
}
174+
175+
const buildCore = spinner();
176+
177+
// Inform the user that the core is being built.
178+
buildCore.start('Building core');
179+
180+
/**
181+
* Build core
182+
* Otherwise, the uncommitted changes will not be reflected in the tests because:
183+
* - popping the stash doesn't trigger a re-render even if `npm start` is running
184+
* - app is not running the `npm start` command
185+
*/
186+
await execAsync('npm run build').catch((error) => {
187+
// Clean up the local ground truths.
188+
cleanUpFiles();
189+
190+
buildCore.stop('Failed to build core');
191+
console.error(error);
192+
return process.exit(0);
193+
});
194+
195+
buildCore.stop('Built core');
196+
197+
const runE2ETests = spinner();
198+
199+
// Inform the user that the E2E tests are being run.
200+
runE2ETests.start('Running E2E tests');
201+
202+
// User provided a component value.
203+
if (componentValue) {
204+
await execAsync(`npm run test.e2e.docker.ci ${componentValue}`).catch((error) => {
205+
// Clean up the local ground truths.
206+
cleanUpFiles();
207+
208+
runE2ETests.stop('Failed to run E2E tests');
209+
console.error(error);
210+
return process.exit(0);
211+
});
212+
} else {
213+
await execAsync('npm run test.e2e.docker.ci').catch((error) => {
214+
// Clean up the local ground truths.
215+
cleanUpFiles();
216+
217+
runE2ETests.stop('Failed to run E2E tests');
218+
console.error(error);
219+
return process.exit(0);
220+
});
221+
}
222+
223+
runE2ETests.stop('Ran E2E tests');
224+
225+
// Clean up the local ground truths.
226+
await cleanUpFiles();
227+
228+
// Ask user if they want to open the Playwright report.
229+
const shouldOpenReport = await confirm({
230+
message: 'Do you want to open the Playwright report?',
231+
});
232+
233+
// User cancelled the operation with `Ctrl+C` or `CMD+C`.
234+
if (isCancel(shouldOpenReport)) {
235+
cancel('Operation cancelled');
236+
return process.exit(0);
237+
}
238+
239+
// User chose to open the Playwright report.
240+
if (shouldOpenReport) {
241+
// Use spawn to display the server information and the key to quit the server.
242+
spawn('npx', ['playwright', 'show-report'], {
243+
stdio: 'inherit',
244+
});
245+
} else {
246+
// Inform the user that the Playwright report can be opened by running the following command.
247+
log.info('If you change your mind, you can open the Playwright report by running the following command:');
248+
log.info(color.bold('npx playwright show-report'));
249+
}
250+
251+
if (shouldOpenReport) {
252+
outro("You're all set! Don't forget to quit serving the Playwright report when you're done.");
253+
} else {
254+
outro("You're all set!");
255+
}
256+
257+
await sleep(1000);
258+
}
259+
260+
main().catch(console.error);

0 commit comments

Comments
 (0)