Skip to content

chore(ci): retry flaky tasks #3697

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 16, 2024
Merged

chore(ci): retry flaky tasks #3697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4f38ad7
chore(ci): retry flaky tasks
millotp Sep 12, 2024
7021071
sleep
millotp Sep 12, 2024
7ef1559
Merge branch 'main' into chore/retry-flaky
millotp Sep 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions .github/workflows/check.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,13 +430,14 @@ jobs:
type: minimal
language: kotlin

- name: remove previous build from linux
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this didn't help

run: |
rm -rf clients/algoliasearch-client-kotlin/build || true
rm -rf clients/algoliasearch-client-kotlin/.kotlin || true
rm -rf clients/algoliasearch-client-kotlin/.gradle || true
- name: Build clients for macOS
id: build
continue-on-error: true
run: yarn cli build clients kotlin

- run: yarn cli build clients kotlin
- name: Retry build
if: ${{ steps.build.outcome == 'failure' }}
run: yarn cli build clients kotlin

- name: Set output
id: setoutput
Expand Down Expand Up @@ -478,7 +479,14 @@ jobs:
language: swift
version: ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).version }}

- run: yarn cli cts run swift ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).toRun }} -v ${{ !contains(format('{0} {1}', github.event.pull_request.title, github.event.head_commit.message), '[skip-e2e]') && '--no-e2e' || '' }}
- name: Run tests on macOS
id: run-test
continue-on-error: true
run: yarn cli cts run swift ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).toRun }} -v ${{ !contains(format('{0} {1}', github.event.pull_request.title, github.event.head_commit.message), '[skip-e2e]') && '--no-e2e' || '' }}

- name: Retry tests
if: ${{ steps.run-test.outcome == 'failure' }}
run: yarn cli cts run swift ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).toRun }} -v ${{ !contains(format('{0} {1}', github.event.pull_request.title, github.event.head_commit.message), '[skip-e2e]') && '--no-e2e' || '' }}

- name: Set output
id: setoutput
Expand Down Expand Up @@ -622,7 +630,7 @@ jobs:

push_and_release:
runs-on: ubuntu-22.04
timeout-minutes: 15
timeout-minutes: 30
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

omgwtf

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

kotlin is taking 12 min to deploy and often fails, so weed need 30 min when it retries

needs:
- codegen
- check_green
Expand Down
35 changes: 28 additions & 7 deletions scripts/ci/codegen/waitForAllReleases.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ type Run = {
language: Language;
run?: components['schemas']['workflow-run'];
finished: boolean;
retried: boolean;
};

async function fetchAllRuns(runs: Run[]): Promise<void> {
Expand All @@ -34,8 +35,8 @@ async function fetchAllRuns(runs: Run[]): Promise<void> {
return;
}

// check that the run was created less than 10 minutes ago
if (Date.now() - Date.parse(workflowRun.data.workflow_runs[0].created_at) > 15 * 60 * 1000) {
// check that the run was created less than 20 minutes ago
if (Date.now() - Date.parse(workflowRun.data.workflow_runs[0].created_at) > 20 * 60 * 1000) {
return;
}

Expand All @@ -44,6 +45,12 @@ async function fetchAllRuns(runs: Run[]): Promise<void> {
);
}

function sleep(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}

async function waitForAllReleases(languagesReleased: Language[]): Promise<void> {
const lastCommitMessage = await run('git log -1 --format="%s"');

Expand All @@ -62,7 +69,7 @@ async function waitForAllReleases(languagesReleased: Language[]): Promise<void>
)
)
.filter((lang) => lang.available)
.map((lang) => ({ language: lang.lang, run: undefined, finished: false }));
.map((lang) => ({ language: lang.lang, run: undefined, finished: false, retried: false }));

console.log(
`Waiting for all releases CI to finish for the following languages: ${runs.map((l) => l.language).join(', ')}`,
Expand All @@ -72,7 +79,7 @@ async function waitForAllReleases(languagesReleased: Language[]): Promise<void>

const start = Date.now();
// kotlin release can take a long time
while (Date.now() - start < 1000 * 60 * 12) {
while (Date.now() - start < 1000 * 60 * 20) {
await fetchAllRuns(runs);
for (const ciRun of runs) {
if (ciRun.finished) {
Expand All @@ -86,6 +93,22 @@ async function waitForAllReleases(languagesReleased: Language[]): Promise<void>

if (ciRun.run.status === 'completed') {
const success = ciRun.run.conclusion === 'success';
if (!success && !ciRun.retried) {
// retry once
console.log(`❌ ${ciRun.language} CI failed, retrying once`);
await getOctokit().actions.reRunWorkflowFailedJobs({
owner: 'algolia',
repo: getClientsConfigField(ciRun.language, 'gitRepoId'),
run_id: ciRun.run.id,
});

// sleep for a bit to let the CI start
await sleep(15000);

ciRun.retried = true;

continue;
}
console.log(`${success ? '✅' : '❌'} ${ciRun.language} CI finished with conclusion: ${ciRun.run.conclusion}`);
if (!success) {
failures.push(ciRun.language);
Expand All @@ -99,9 +122,7 @@ async function waitForAllReleases(languagesReleased: Language[]): Promise<void>
break;
}

await new Promise((resolve) => {
setTimeout(resolve, 15000);
});
await sleep(15000);
}

if (failures.length > 0) {
Expand Down
Loading