chore(ci): retry flaky tasks (#3697)

Author: millotp

.github/workflows/check.yml

@@ -430,13 +430,14 @@ jobs:
           type: minimal
           language: kotlin
 
-      - name: remove previous build from linux
-        run: |
-          rm -rf clients/algoliasearch-client-kotlin/build || true
-          rm -rf clients/algoliasearch-client-kotlin/.kotlin || true
-          rm -rf clients/algoliasearch-client-kotlin/.gradle || true
+      - name: Build clients for macOS
+        id: build
+        continue-on-error: true
+        run: yarn cli build clients kotlin
 
-      - run: yarn cli build clients kotlin
+      - name: Retry build
+        if: ${{ steps.build.outcome == 'failure' }}
+        run: yarn cli build clients kotlin
 
       - name: Set output
         id: setoutput
@@ -478,7 +479,14 @@ jobs:
           language: swift
           version: ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).version }}
 
-      - run: yarn cli cts run swift ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).toRun }} -v ${{ !contains(format('{0} {1}', github.event.pull_request.title, github.event.head_commit.message), '[skip-e2e]') && '--no-e2e' || '' }}
+      - name: Run tests on macOS
+        id: run-test
+        continue-on-error: true
+        run: yarn cli cts run swift ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).toRun }} -v ${{ !contains(format('{0} {1}', github.event.pull_request.title, github.event.head_commit.message), '[skip-e2e]') && '--no-e2e' || '' }}
+
+      - name: Retry tests
+        if: ${{ steps.run-test.outcome == 'failure' }}
+        run: yarn cli cts run swift ${{ fromJSON(needs.setup.outputs.SWIFT_DATA).toRun }} -v ${{ !contains(format('{0} {1}', github.event.pull_request.title, github.event.head_commit.message), '[skip-e2e]') && '--no-e2e' || '' }}
 
       - name: Set output
         id: setoutput
@@ -622,7 +630,7 @@ jobs:
 
   push_and_release:
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     needs:
       - codegen
       - check_green

scripts/ci/codegen/waitForAllReleases.ts

@@ -12,6 +12,7 @@ type Run = {
   language: Language;
   run?: components['schemas']['workflow-run'];
   finished: boolean;
+  retried: boolean;
 };
 
 async function fetchAllRuns(runs: Run[]): Promise<void> {
@@ -34,8 +35,8 @@ async function fetchAllRuns(runs: Run[]): Promise<void> {
           return;
         }
 
-        // check that the run was created less than 10 minutes ago
-        if (Date.now() - Date.parse(workflowRun.data.workflow_runs[0].created_at) > 15 * 60 * 1000) {
+        // check that the run was created less than 20 minutes ago
+        if (Date.now() - Date.parse(workflowRun.data.workflow_runs[0].created_at) > 20 * 60 * 1000) {
           return;
         }
 
@@ -44,6 +45,12 @@ async function fetchAllRuns(runs: Run[]): Promise<void> {
   );
 }
 
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
 async function waitForAllReleases(languagesReleased: Language[]): Promise<void> {
   const lastCommitMessage = await run('git log -1 --format="%s"');
 
@@ -62,7 +69,7 @@ async function waitForAllReleases(languagesReleased: Language[]): Promise<void>
     )
   )
     .filter((lang) => lang.available)
-    .map((lang) => ({ language: lang.lang, run: undefined, finished: false }));
+    .map((lang) => ({ language: lang.lang, run: undefined, finished: false, retried: false }));
 
   console.log(
     `Waiting for all releases CI to finish for the following languages: ${runs.map((l) => l.language).join(', ')}`,
@@ -72,7 +79,7 @@ async function waitForAllReleases(languagesReleased: Language[]): Promise<void>
 
   const start = Date.now();
   // kotlin release can take a long time
-  while (Date.now() - start < 1000 * 60 * 12) {
+  while (Date.now() - start < 1000 * 60 * 20) {
     await fetchAllRuns(runs);
     for (const ciRun of runs) {
       if (ciRun.finished) {
@@ -86,6 +93,22 @@ async function waitForAllReleases(languagesReleased: Language[]): Promise<void>
 
       if (ciRun.run.status === 'completed') {
         const success = ciRun.run.conclusion === 'success';
+        if (!success && !ciRun.retried) {
+          // retry once
+          console.log(`❌ ${ciRun.language} CI failed, retrying once`);
+          await getOctokit().actions.reRunWorkflowFailedJobs({
+            owner: 'algolia',
+            repo: getClientsConfigField(ciRun.language, 'gitRepoId'),
+            run_id: ciRun.run.id,
+          });
+
+          // sleep for a bit to let the CI start
+          await sleep(15000);
+
+          ciRun.retried = true;
+
+          continue;
+        }
         console.log(`${success ? '✅' : '❌'} ${ciRun.language} CI finished with conclusion: ${ciRun.run.conclusion}`);
         if (!success) {
           failures.push(ciRun.language);
@@ -99,9 +122,7 @@ async function waitForAllReleases(languagesReleased: Language[]): Promise<void>
       break;
     }
 
-    await new Promise((resolve) => {
-      setTimeout(resolve, 15000);
-    });
+    await sleep(15000);
   }
 
   if (failures.length > 0) {