fix(cts): split requests and e2e tests

generators/src/main/java/com/algolia/codegen/cts/manager/PythonCTSManager.java

@@ -18,6 +18,7 @@ public void addTestsSupportingFiles(List<SupportingFile> supportingFiles) {
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/", "__init__.py"));
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/requests", "__init__.py"));
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/client", "__init__.py"));
+    supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/e2e", "__init__.py"));
   }
 
   @Override

generators/src/main/java/com/algolia/codegen/cts/tests/TestsRequest.java

@@ -15,6 +15,7 @@
 public class TestsRequest extends TestsGenerator {
 
   private final boolean withSnippets;
+  private List<SupportingFile> supportingFiles;
 
   public TestsRequest(String language, String client, boolean withSnippets) {
     super(language, client);
@@ -64,13 +65,40 @@ public void addSupportingFiles(List<SupportingFile> supportingFiles, String outp
     if (!available()) {
       return;
     }
+
+    this.supportingFiles = supportingFiles;
+
     supportingFiles.add(
       new SupportingFile(
         "tests/requests/requests.mustache",
         "tests/output/" + language + "/" + outputFolder + "/requests",
         Helpers.createClientName(client, language) + extension
       )
     );
+    if (new File("templates/" + language + "/tests/e2e/e2e.mustache").exists()) {
+      supportingFiles.add(
+        new SupportingFile(
+          "tests/e2e/e2e.mustache",
+          "tests/output/" + language + "/" + outputFolder + "/e2e",
+          Helpers.createClientName(client, language) + extension
+        )
+      );
+    }
+  }
+
+  private String escapeBody(String body) {
+    if (body == null) {
+      return null;
+    }
+
+    switch (language) {
+      case "go": // jsonassert expect % to be formatted, we need to escape them
+        return body.replace("%", "%%");
+      case "dart": // Same thing but for $
+        return body.replace("$", "\\$");
+      default:
+        return body;
+    }
   }
 
   @Override
@@ -82,6 +110,7 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
     }
 
     List<Object> blocks = new ArrayList<>();
+    List<Object> blocksE2E = new ArrayList<>();
     ParametersWithDataType paramsType = new ParametersWithDataType(models, language, client);
 
     bundle.put("e2eApiKey", client.equals("monitoring") ? "MONITORING_API_KEY" : "ALGOLIA_ADMIN_KEY");
@@ -138,15 +167,7 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
               req.request.body = "{}";
             }
 
-            // For golang, jsonassert expect % to be formatted, we need to escape them
-            if (language.equals("go") && req.request.body != null) {
-              req.request.body = req.request.body.replace("%", "%%");
-            }
-
-            // For dart, same thing but for $
-            if (language.equals("dart") && req.request.body != null) {
-              req.request.body = req.request.body.replace("$", "\\$");
-            }
+            req.request.body = escapeBody(req.request.body);
 
             // In a case of a `GET` or `DELETE` request, we want to assert if the body
             // is correctly parsed (absent from the payload)
@@ -156,7 +177,7 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
           }
 
           if (req.response != null) {
-            bundle.put("hasE2E", true);
+            req.response.body = escapeBody(req.response.body);
             test.put("response", req.response);
           }
 
@@ -219,7 +240,21 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
       }
 
       blocks.add(testObj);
+
+      // extract e2e
+      List<Map<String, Object>> e2e = tests.stream().filter(t -> t.get("response") != null).toList();
+      if (e2e.size() > 0) {
+        Map<String, Object> e2eObj = new HashMap<>();
+        e2eObj.put("tests", e2e);
+        e2eObj.put("operationId", operationId);
+        blocksE2E.add(e2eObj);
+      }
     }
     bundle.put("blocksRequests", blocks);
+    if (!blocksE2E.isEmpty()) {
+      bundle.put("blocksE2E", blocksE2E);
+    } else if (supportingFiles != null) {
+      supportingFiles.removeIf(f -> f.getTemplateFile().equals("tests/e2e/e2e.mustache"));
+    }
   }
 }

scripts/ci/githubActions/createMatrix.ts

@@ -69,15 +69,15 @@ async function createClientMatrix(baseBranch: string): Promise<void> {
     const testsRootFolder = `tests/output/${language}`;
     const testsOutputBase = `${testsRootFolder}/${getTestOutputFolder(language)}`;
     // We delete tests to ensure the CI only run tests against what changed.
-    const testsToDelete = `${testsOutputBase}/client ${testsOutputBase}/requests`;
+    const testsToDelete = `${testsOutputBase}/client ${testsOutputBase}/requests ${testsOutputBase}/e2e`;
 
     // We only store tests of clients that ran during this job, the rest stay as is
     let testsToStore = matrix[language].toRun
       .map((client) => {
         const clientName = createClientName(client, language);
         const extension = getTestExtension(language);
 
-        return `${testsOutputBase}/client/${clientName}${extension} ${testsOutputBase}/requests/${clientName}${extension}`;
+        return `${testsOutputBase}/client/${clientName}${extension} ${testsOutputBase}/requests/${clientName}${extension} ${testsOutputBase}/e2e/${clientName}${extension}`;
       })
       .join(' ');
 

scripts/cts/runCts.ts

@@ -36,9 +36,12 @@ async function runCtsOne(language: string): Promise<void> {
       break;
     case 'php':
       await runComposerInstall();
-      await run(`php ./clients/algoliasearch-client-php/vendor/bin/phpunit ${cwd}`, {
-        language,
-      });
+      await run(
+        `php ./clients/algoliasearch-client-php/vendor/bin/phpunit --testdox --fail-on-warning ${cwd}`,
+        {
+          language,
+        },
+      );
       break;
     case 'python':
       await run('poetry lock --no-update && poetry install --sync && poetry run pytest -vv', {

scripts/husky/pre-commit.mjs

@@ -22,6 +22,7 @@ export function getPatterns() {
   for (const [language, { tests }] of Object.entries(clientConfig)) {
     entries.unshift(`tests/output/${language}/${tests.outputFolder}/client/**`);
     entries.unshift(`tests/output/${language}/${tests.outputFolder}/requests/**`);
+    entries.unshift(`tests/output/${language}/${tests.outputFolder}/e2e/**`);
   }
   return entries;
 }

templates/csharp/tests/e2e/e2e.mustache

@@ -0,0 +1,73 @@
+// {{generationBanner}}
+using Algolia.Search.Http;
+using Algolia.Search.Clients;
+using Algolia.Search.Models.{{clientPrefix}};
+using Algolia.Search.Serializer;
+using Algolia.Search.Tests.Utils;
+using Xunit;
+using System.Text.Json;
+using Quibble.Xunit;
+using dotenv.net;
+using Action = Algolia.Search.Models.Search.Action;
+
+public class {{client}}RequestTestsE2E
+{
+private readonly {{client}} _client;
+
+  public {{client}}RequestTestsE2E()
+  {
+    DotEnv.Load(options: new DotEnvOptions(ignoreExceptions: true, probeForEnv: true, probeLevelsToSearch: 8, envFilePaths: new[] { ".env" }));
+
+    var appId = Environment.GetEnvironmentVariable("ALGOLIA_APPLICATION_ID");
+    if (appId == null)
+    {
+      throw new Exception("please provide an `ALGOLIA_APPLICATION_ID` env var for e2e tests");
+    }
+
+    var apiKey = Environment.GetEnvironmentVariable("{{e2eApiKey}}");
+    if (apiKey == null)
+    {
+      throw new Exception("please provide an `{{e2eApiKey}}` env var for e2e tests");
+    }
+
+    _client = new {{client}}(new {{clientPrefix}}Config(appId, apiKey{{#hasRegionalHost}},"{{defaultRegion}}"{{/hasRegionalHost}}));
+  }
+
+  [Fact]
+  public void Dispose()
+  {
+
+  }
+
+  {{#blocksE2E}}
+  {{#tests}}
+  [Fact(DisplayName = "{{{testName}}}")]
+  public async Task {{#lambda.pascalcase}}{{method}}Test{{testIndex}}{{/lambda.pascalcase}}()
+  {
+      try {
+        var resp = await _client.{{#lambda.pascalcase}}{{method}}{{/lambda.pascalcase}}Async{{#isGeneric}}<Hit>{{/isGeneric}}({{#parametersWithDataType}}{{> tests/generateParams}}{{^-last}},{{/-last}}{{/parametersWithDataType}}{{#hasRequestOptions}}, new RequestOptions(){
+          {{#requestOptions.queryParameters}}
+            QueryParameters = new Dictionary<string, object>(){ {{#parametersWithDataType}} {"{{{key}}}", {{> tests/requests/requestOptionsParams}} } {{^-last}},{{/-last}}{{/parametersWithDataType}} },
+          {{/requestOptions.queryParameters}}
+          {{#requestOptions.headers}}
+            Headers = new Dictionary<string, string>(){ {{#parametersWithDataType}} {"{{{key}}}", "{{{value}}}" } {{^-last}},{{/-last}}{{/parametersWithDataType}} },
+          {{/requestOptions.headers}}
+        }{{/hasRequestOptions}});
+      {{#response}}
+      {{#statusCode}}
+          // Check status code {{statusCode}}
+          Assert.NotNull(resp);
+      {{/statusCode}}
+
+      {{#body}}
+          JsonAssert.EqualOverrideDefault("{{#lambda.escapeQuotes}}{{{.}}}{{/lambda.escapeQuotes}}", JsonSerializer.Serialize(resp, JsonConfig.Options), new JsonDiffConfig(true));
+      {{/body}}
+      } catch (Exception e)
+      {
+        Assert.Fail("An exception was thrown: " + e.Message);
+      }
+    {{/response}}
+  }
+  {{/tests}}
+  {{/blocksE2E}}
+}

templates/csharp/tests/requests/requests.mustache

@@ -12,31 +12,13 @@ using Action = Algolia.Search.Models.Search.Action;
 
 public class {{client}}RequestTests
 {
-private readonly {{client}} _client{{#hasE2E}}, _e2eClient{{/hasE2E}};
+private readonly {{client}} _client;
   private readonly EchoHttpRequester _echo;
 
   public {{client}}RequestTests()
   {
     _echo = new EchoHttpRequester();
     _client = new {{client}}(new {{clientPrefix}}Config("appId", "apiKey"{{#hasRegionalHost}},"{{defaultRegion}}"{{/hasRegionalHost}}), _echo);
-
-    {{#hasE2E}}
-      DotEnv.Load(options: new DotEnvOptions(ignoreExceptions: true, probeForEnv: true, probeLevelsToSearch: 8, envFilePaths: new[] { ".env" }));
-
-      var e2EAppId = Environment.GetEnvironmentVariable("ALGOLIA_APPLICATION_ID");
-      if (e2EAppId == null)
-      {
-        throw new Exception("please provide an `ALGOLIA_APPLICATION_ID` env var for e2e tests");
-      }
-
-      var e2EApiKey = Environment.GetEnvironmentVariable("{{e2eApiKey}}");
-      if (e2EApiKey == null)
-      {
-        throw new Exception("please provide an `{{e2eApiKey}}` env var for e2e tests");
-      }
-
-      _e2eClient = new {{client}}(new {{clientPrefix}}Config(e2EAppId, e2EApiKey{{#hasRegionalHost}},"{{defaultRegion}}"{{/hasRegionalHost}}));
-    {{/hasE2E}}
   }
 
   [Fact]
@@ -99,32 +81,6 @@ private readonly {{client}} _client{{#hasE2E}}, _e2eClient{{/hasE2E}};
         }
     {{/headers}}
     {{/request}}
-
-
-    {{#response}}
-      // e2e
-      try {
-        var resp = await _e2eClient.{{#lambda.pascalcase}}{{method}}{{/lambda.pascalcase}}Async{{#isGeneric}}<Hit>{{/isGeneric}}({{#parametersWithDataType}}{{> tests/generateParams}}{{^-last}},{{/-last}}{{/parametersWithDataType}}{{#hasRequestOptions}}, new RequestOptions(){
-          {{#requestOptions.queryParameters}}
-            QueryParameters = new Dictionary<string, object>(){ {{#parametersWithDataType}} {"{{{key}}}", {{> tests/requests/requestOptionsParams}} } {{^-last}},{{/-last}}{{/parametersWithDataType}} },
-          {{/requestOptions.queryParameters}}
-          {{#requestOptions.headers}}
-            Headers = new Dictionary<string, string>(){ {{#parametersWithDataType}} {"{{{key}}}", "{{{value}}}" } {{^-last}},{{/-last}}{{/parametersWithDataType}} },
-          {{/requestOptions.headers}}
-        }{{/hasRequestOptions}});
-      {{#statusCode}}
-          // Check status code {{statusCode}}
-          Assert.NotNull(resp);
-      {{/statusCode}}
-
-      {{#body}}
-          JsonAssert.EqualOverrideDefault("{{#lambda.escapeQuotes}}{{{.}}}{{/lambda.escapeQuotes}}", JsonSerializer.Serialize(resp, JsonConfig.Options), new JsonDiffConfig(true));
-      {{/body}}
-      } catch (Exception e)
-      {
-        Assert.Fail("An exception was thrown: " + e.Message);
-      }
-    {{/response}}
   }
   {{/tests}}
   {{/blocksRequests}}

templates/go/tests/e2e/e2e.mustache

@@ -0,0 +1,69 @@
+// {{generationBanner}}
+package requestse2e
+
+import (
+  "os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/joho/godotenv"
+
+  "github.com/algolia/algoliasearch-client-go/v4/algolia/{{clientImport}}"
+)
+
+func createE2E{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}Client(t *testing.T) *{{clientPrefix}}.APIClient {
+  t.Helper()
+
+  appID := os.Getenv("ALGOLIA_APPLICATION_ID")
+  if appID == "" && os.Getenv("CI") != "true" {
+    err := godotenv.Load("../../../../.env")
+    require.NoError(t, err)
+    appID = os.Getenv("ALGOLIA_APPLICATION_ID")
+  }
+  apiKey := os.Getenv("{{e2eApiKey}}")
+  client, err := {{clientPrefix}}.NewClient(appID, apiKey, {{#hasRegionalHost}}{{clientPrefix}}.{{#lambda.uppercase}}{{defaultRegion}}{{/lambda.uppercase}},{{/hasRegionalHost}})
+  require.NoError(t, err)
+
+  return client
+}
+
+{{#blocksE2E}}
+func Test{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}E2E_{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}(t *testing.T) {
+    {{#tests}}
+    t.Run("{{{testName}}}", func(t *testing.T) {
+      client := createE2E{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}Client(t)
+      res, err := client.{{#lambda.titlecase}}{{method}}{{/lambda.titlecase}}({{#hasOperationParams}}client.NewApi{{#lambda.titlecase}}{{method}}{{/lambda.titlecase}}Request(
+        {{#parametersWithDataType}}{{#required}}{{> tests/generateParams}},{{/required}}{{/parametersWithDataType}}
+      ){{#parametersWithDataType}}{{^required}}.With{{#lambda.pascalcase}}{{{key}}}{{/lambda.pascalcase}}({{> tests/generateParams}}){{/required}}{{/parametersWithDataType}}{{/hasOperationParams}}{{#requestOptions}}{{#hasOperationParams}},{{/hasOperationParams}}
+      {{#queryParameters.parametersWithDataType}}{{clientPrefix}}.QueryParamOption("{{{key}}}", {{> tests/generateInnerParams}}),{{/queryParameters.parametersWithDataType}}{{#headers.parametersWithDataType}}{{clientPrefix}}.HeaderParamOption("{{{key}}}", {{> tests/generateInnerParams}}),{{/headers.parametersWithDataType}}
+      {{/requestOptions}})
+      require.NoError(t, err)
+      _ = res
+
+      {{#response}}
+      {{#body}}
+      rawBody, err := json.Marshal(res)
+      require.NoError(t, err)
+
+      var rawBodyMap any
+      err = json.Unmarshal(rawBody, &rawBodyMap)
+      require.NoError(t, err)
+
+      expectedBodyRaw := `{{{.}}}`
+      var expectedBody any
+      err = json.Unmarshal([]byte(expectedBodyRaw), &expectedBody)
+      require.NoError(t, err)
+
+      unionBody := tests.Union(expectedBody, rawBodyMap)
+      unionBodyRaw, err := json.Marshal(unionBody)
+      require.NoError(t, err)
+
+      jsonassert.New(t).Assertf(string(unionBodyRaw), expectedBodyRaw)
+      {{/body}}
+      {{/response}}
+    })
+{{/tests}}
+}
+
+{{/blocksE2E}}