fix(cts): retry e2e tests (#3341)

Author: millotp

.github/workflows/check.yml

@@ -242,7 +242,13 @@ jobs:
           key: node-modules-tests-${{ hashFiles('tests/output/javascript/yarn.lock') }}
 
       - name: Run CTS
-        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }}
+        id: cts
+        continue-on-error: true
+        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} ${{ github.event.pull_request.head.repo.fork && '--exclude-e2e' || '' }}
+
+      - name: Retry e2e CTS
+        if: ${{ !github.event.pull_request.head.repo.fork && github.event.number && steps.cts.outcome == 'failure' }}
+        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --exclude-unit
 
       - name: Generate code snippets for documentation
         run: yarn cli snippets javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }}
@@ -326,7 +332,13 @@ jobs:
         run: yarn cli cts generate ${{ matrix.client.language }} ${{ matrix.client.toRun }}
 
       - name: Run CTS
-        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }}
+        id: cts
+        continue-on-error: true
+        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} ${{ github.event.pull_request.head.repo.fork && '--exclude-e2e' || '' }}
+
+      - name: Retry e2e CTS
+        if: ${{ !github.event.pull_request.head.repo.fork && steps.cts.outcome == 'failure' }}
+        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --exclude-unit
 
       - name: Generate code snippets for documentation
         run: yarn cli snippets ${{ matrix.client.language }} ${{ matrix.client.toRun }}
@@ -522,6 +534,8 @@ jobs:
       - check_green
     if: |
       always() &&
+      !contains(needs.*.result, 'cancelled') &&
+      !contains(needs.*.result, 'failure') &&
       github.ref == 'refs/heads/main'
     permissions:
       pull-requests: write

netlify.toml

@@ -3,4 +3,4 @@
   publish="website/build"
 
 [build.environment]
-  YARN_VERSION = "3.5.0"
+  YARN_VERSION = "4.3.1"

scripts/cli/index.ts

@@ -129,16 +129,29 @@ ctsCommand
   .addArgument(args.language)
   .addArgument(args.clients)
   .option(flags.verbose.flag, flags.verbose.description)
-  .action(async (langArg: LangArg, clientArg: string[], { verbose }) => {
-    const { language, client } = transformSelection({
-      langArg,
-      clientArg,
-    });
-
-    setVerbose(Boolean(verbose));
-
-    await runCts(language === ALL ? LANGUAGES : [language], client);
-  });
+  .option('-e, --exclude-e2e', "don't run the e2e tests, useful for offline testing")
+  .option('-u, --exclude-unit', "don't run the client and requests tests")
+  .action(
+    async (
+      langArg: LangArg,
+      clientArg: string[],
+      { verbose, excludeE2e: excludeE2E, excludeUnit },
+    ) => {
+      const { language, client } = transformSelection({
+        langArg,
+        clientArg,
+      });
+
+      setVerbose(Boolean(verbose));
+
+      await runCts(
+        language === ALL ? LANGUAGES : [language],
+        client,
+        Boolean(excludeE2E),
+        Boolean(excludeUnit),
+      );
+    },
+  );
 
 ctsCommand
   .command('server')

scripts/cts/runCts.ts

@@ -7,47 +7,86 @@ import { assertChunkWrapperValid } from './testServer/chunkWrapper.js';
 import { assertValidReplaceAllObjects } from './testServer/replaceAllObjects.js';
 import { assertValidTimeouts } from './testServer/timeout.js';
 
-async function runCtsOne(language: string): Promise<void> {
-  const spinner = createSpinner(`running cts for '${language}'`);
+async function runCtsOne(
+  language: Language,
+  excludeE2E: boolean,
+  excludeUnit: boolean,
+): Promise<void> {
   const cwd = `tests/output/${language}`;
+
+  const folders: string[] = [];
+  if (language !== 'dart' && language !== 'kotlin' && !excludeE2E) {
+    folders.push('e2e');
+  }
+  if (!excludeUnit) {
+    folders.push('client', 'requests');
+  }
+
+  const spinner = createSpinner(`running cts for '${language}' in folder(s) ${folders.join(', ')}`);
+
+  if (folders.length === 0) {
+    spinner.succeed(`skipping '${language}' because all tests are excluded`);
+    return;
+  }
+
+  const filter = (mapper: (f: string) => string): string => folders.map(mapper).join(' ');
+
   switch (language) {
     case 'csharp':
-      await run('dotnet test /clp:ErrorsOnly', { cwd, language });
+      await run(
+        `dotnet test /clp:ErrorsOnly --filter 'Algolia.Search.Tests${folders.map((f) => `|Algolia.Search.${f}`).join('')}'`,
+        { cwd, language },
+      );
       break;
     case 'dart':
-      await run('dart test', { cwd, language });
-      break;
-    case 'go':
-      await run(`go test -race -count 1 ${isVerbose() ? '-v' : ''} ./...`, {
+      await run(`dart test ${filter((f) => `test/${f}`)}`, {
         cwd,
         language,
       });
       break;
+    case 'go':
+      await run(
+        `go test -race -count 1 ${isVerbose() ? '-v' : ''} ${filter((f) => `gotests/tests/${f}/...`)}`,
+        {
+          cwd,
+          language,
+        },
+      );
+      break;
     case 'java':
-      await run('./gradle/gradlew -p tests/output/java test --rerun', { language });
+      await run(
+        `./gradle/gradlew -p tests/output/java test --rerun ${filter((f) => `--tests 'com.algolia.${f}*'`)}`,
+        { language },
+      );
       break;
     case 'javascript':
-      await run('YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install && yarn test', {
-        cwd,
-      });
+      await run(
+        `YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install && yarn test ${filter((f) => `dist/${f}`)}`,
+        {
+          cwd,
+        },
+      );
       break;
     case 'kotlin':
       await run('./gradle/gradlew -p tests/output/kotlin allTests', { language });
       break;
     case 'php':
       await runComposerInstall();
       await run(
-        `php ./clients/algoliasearch-client-php/vendor/bin/phpunit --testdox --fail-on-warning ${cwd}`,
+        `php ./clients/algoliasearch-client-php/vendor/bin/phpunit --testdox --fail-on-warning ${filter((f) => `${cwd}/src/${f}`)}`,
         {
           language,
         },
       );
       break;
     case 'python':
-      await run('poetry lock --no-update && poetry install --sync && poetry run pytest -vv', {
-        cwd,
-        language,
-      });
+      await run(
+        `poetry lock --no-update && poetry install --sync && poetry run pytest -vv ${filter((f) => `tests/${f}`)}`,
+        {
+          cwd,
+          language,
+        },
+      );
       break;
     case 'ruby':
       await run(`bundle install && bundle exec rake test --trace`, {
@@ -56,14 +95,20 @@ async function runCtsOne(language: string): Promise<void> {
       });
       break;
     case 'scala':
-      await run('sbt test', { cwd, language });
-      break;
-    case 'swift':
-      await run('swift test -Xswiftc -suppress-warnings -q --parallel', {
+      await run(`sbt test`, {
         cwd,
         language,
       });
       break;
+    case 'swift':
+      await run(
+        `swift test -Xswiftc -suppress-warnings -q --parallel ${filter((f) => `--filter ${f}.*`)}`,
+        {
+          cwd,
+          language,
+        },
+      );
+      break;
     default:
       spinner.warn(`skipping unknown language '${language}' to run the CTS`);
       return;
@@ -72,14 +117,19 @@ async function runCtsOne(language: string): Promise<void> {
 }
 
 // the clients option is only used to determine if we need to start the test server, it will run the tests for all clients anyway.
-export async function runCts(languages: Language[], clients: string[]): Promise<void> {
-  const useTestServer = clients.includes('search') || clients.includes('all');
+export async function runCts(
+  languages: Language[],
+  clients: string[],
+  excludeE2E: boolean,
+  excludeUnit: boolean,
+): Promise<void> {
+  const useTestServer = !excludeUnit && (clients.includes('search') || clients.includes('all'));
   let close: () => Promise<void> = async () => {};
   if (useTestServer) {
     close = await startTestServer();
   }
   for (const lang of languages) {
-    await runCtsOne(lang);
+    await runCtsOne(lang, excludeE2E, excludeUnit);
   }
 
   if (useTestServer) {

scripts/cts/testServer/timeout.ts

@@ -1,15 +1,12 @@
 import type { Server } from 'http';
 
+import { expect } from 'chai';
 import type express from 'express';
 
 import { setupServer } from '.';
 
 const timeoutState: Record<string, { timestamp: number[]; duration: number[] }> = {};
 
-function aboutEqual(a: number, b: number, epsilon = 100): boolean {
-  return Math.abs(a - b) <= epsilon;
-}
-
 export function assertValidTimeouts(expectedCount: number): void {
   // assert that the retry strategy uses the correct timings, by checking the time between each request, and how long each request took before being timed out
   // there should be no delay between requests, only an increase in timeout.
@@ -18,37 +15,22 @@ export function assertValidTimeouts(expectedCount: number): void {
   }
 
   for (const [lang, state] of Object.entries(timeoutState)) {
-    if (state.timestamp.length !== 3 || state.duration.length !== 3) {
-      throw new Error(`Expected 3 requests for ${lang}, got ${state.timestamp.length}`);
-    }
-
-    let delay = state.timestamp[1] - state.timestamp[0];
-    if (!aboutEqual(delay, state.duration[0])) {
-      throw new Error(`Expected no delay between requests for ${lang}, got ${delay}ms`);
-    }
-
-    delay = state.timestamp[2] - state.timestamp[1];
-    if (!aboutEqual(delay, state.duration[1])) {
-      throw new Error(`Expected no delay between requests for ${lang}, got ${delay}ms`);
-    }
+    expect(state.timestamp.length).to.equal(3);
+    expect(state.duration.length).to.equal(3);
+    expect(state.timestamp[1] - state.timestamp[0]).to.be.closeTo(state.duration[0], 100);
+    expect(state.timestamp[2] - state.timestamp[1]).to.be.closeTo(state.duration[1], 100);
 
     // languages are not consistent yet for the delay between requests
     switch (lang) {
       case 'JavaScript':
-        if (!aboutEqual(state.duration[0] * 4, state.duration[1], 200)) {
-          throw new Error(`Expected increasing delay between requests for ${lang}`);
-        }
+        expect(state.duration[0] * 4).to.be.closeTo(state.duration[1], 200);
         break;
       case 'PHP':
-        if (!aboutEqual(state.duration[0] * 2, state.duration[1], 200)) {
-          throw new Error(`Expected increasing delay between requests for ${lang}`);
-        }
+        expect(state.duration[0] * 2).to.be.closeTo(state.duration[1], 200);
         break;
       default:
         // the delay should be the same, because the `retryCount` is per host instead of global
-        if (!aboutEqual(state.duration[0], state.duration[1])) {
-          throw new Error(`Expected the same delay between requests for ${lang}`);
-        }
+        expect(state.duration[0]).to.be.closeTo(state.duration[1], 100);
         break;
     }
   }

templates/csharp/tests/client/suite.mustache

@@ -11,6 +11,8 @@ using System.Text.Json;
 using Xunit;
 using Quibble.Xunit;
 
+namespace Algolia.Search.client;
+
 public class {{client}}Tests
 {
 private readonly EchoHttpRequester _echo;
@@ -69,4 +71,4 @@ public void Dispose()
     }
   {{/tests}}
 {{/blocksClient}}
-  }
+  }

templates/csharp/tests/e2e/e2e.mustache

@@ -10,6 +10,8 @@ using Quibble.Xunit;
 using dotenv.net;
 using Action = Algolia.Search.Models.Search.Action;
 
+namespace Algolia.Search.e2e;
+
 public class {{client}}RequestTestsE2E
 {
 private readonly {{client}} _client;

templates/csharp/tests/requests/requests.mustache

@@ -10,6 +10,8 @@ using Quibble.Xunit;
 using dotenv.net;
 using Action = Algolia.Search.Models.Search.Action;
 
+namespace Algolia.Search.requests;
+
 public class {{client}}RequestTests
 {
 private readonly {{client}} _client;
@@ -84,4 +86,4 @@ private readonly {{client}} _client;
   }
   {{/tests}}
   {{/blocksRequests}}
-}
+}

templates/go/tests/e2e/e2e.mustache

@@ -1,5 +1,5 @@
 // {{generationBanner}}
-package requestse2e
+package e2e
 
 import (
   "os"

templates/java/tests/e2e/e2e.mustache

@@ -1,4 +1,4 @@
-package com.algolia.methods.e2e;
+package com.algolia.e2e;
 
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 

templates/java/tests/requests/requests.mustache

@@ -1,4 +1,4 @@
-package com.algolia.methods.requests;
+package com.algolia.requests;
 
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -102,4 +102,4 @@ class {{client}}RequestsTests {
     }
     {{/tests}}
     {{/blocksRequests}}
-}
+}

templates/javascript/tests/package.mustache

@@ -2,7 +2,7 @@
   "name": "javascript-tests",
   "version": "1.0.0",
   "scripts": {
-    "test": "rm -rf dist || true && tsc && jest dist --passWithNoTests"
+    "test": "rm -rf dist || true && tsc && jest --passWithNoTests"
   },
   "dependencies": {
     {{#packageDependencies}}

templates/kotlin/tests/requests/requests.mustache

@@ -1,5 +1,5 @@
 // {{generationBanner}}
-package com.algolia.methods.requests
+package com.algolia.requests
 
 import com.algolia.client.api.{{client}}
 import com.algolia.client.model.{{import}}.*