Merge branch 'main' into docs/analytics/update_max_pagination_query_limit

Author: febeck

.eslintrc.cjs

@@ -209,6 +209,17 @@ module.exports = {
 
       plugins: ['@vitest/eslint-plugin'],
     },
+    {
+      files: ['guides/**/*.ts'],
+
+      rules: {
+        'no-console': 0,
+
+        // TODO: remove this after https://algolia.atlassian.net/browse/DI-2984
+        '@typescript-eslint/no-unused-vars': 0,
+        '@typescript-eslint/explicit-function-return-type': 0,
+      },
+    },
     {
       files: ['*.json'],
       parserOptions: {

.github/actions/setup/action.yml

@@ -45,7 +45,7 @@ runs:
 
     - name: Install JavaScript dependencies
       shell: bash
-      run: YARN_ENABLE_HARDENED_MODE=0 yarn install
+      run: YARN_ENABLE_HARDENED_MODE=0 YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install
 
     - name: Build scripts
       shell: bash

.github/workflows/check.yml

@@ -174,7 +174,7 @@ jobs:
           path: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX).bundledPath }}
 
   client_gen_javascript:
-    timeout-minutes: 10
+    timeout-minutes: 15
     runs-on: ubuntu-22.04
     needs:
       - setup
@@ -275,11 +275,20 @@ jobs:
       - name: Run benchmarks
         run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --benchmark --no-client --no-requests --no-e2e
 
-      - name: Generate code snippets for documentation
+      - name: Generate code snippets
         run: yarn cli snippets javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }}
+      
+      - name: Build the snippets to check validity
+        run: yarn cli build snippets javascript
+
+      - name: Generate code guides
+        run: yarn cli guides javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }}
+
+      - name: Build the guides to check validity
+        run: yarn cli build guides javascript
 
       - name: Zip artifact before storing
-        run: zip -r -y clients-javascript.zip clients/algoliasearch-client-javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).testsToStore }} ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).snippetsToStore }} -x "**/node_modules**" "**/.yarn/cache/**" "**/.yarn/install-state.gz" "**/build/**" "**/dist/**" "**/.gradle/**" "**/bin/**" "**/.nx/**"
+        run: zip -r -y clients-javascript.zip clients/algoliasearch-client-javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).testsToStore }} ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).guidesToStore }} -x "**/node_modules**" "**/.yarn/cache/**" "**/.yarn/install-state.gz" "**/build/**" "**/dist/**" "**/.gradle/**" "**/bin/**" "**/.nx/**"
 
       - name: Store javascript clients
         uses: actions/upload-artifact@v4
@@ -381,15 +390,21 @@ jobs:
         if: ${{ matrix.client.isMainVersion }}
         run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --benchmark --no-client --no-requests --no-e2e
 
-      - name: Generate code snippets for documentation
+      - name: Generate code snippets
         run: yarn cli snippets ${{ matrix.client.language }} ${{ matrix.client.toRun }}
 
       - name: Build the snippets to check validity
         run: yarn cli build snippets ${{ matrix.client.language }}
 
+      - name: Generate code guides
+        run: yarn cli guides ${{ matrix.client.language }} ${{ matrix.client.toRun }}
+
+      - name: Build the guides to check validity
+        run: yarn cli build guides ${{ matrix.client.language }}
+
       - name: Zip artifact before storing
         if: ${{ matrix.client.isMainVersion }}
-        run: zip -r -y clients-${{ matrix.client.language }}.zip ${{ matrix.client.path }} ${{ matrix.client.testsToStore }} ${{ matrix.client.snippetsToStore }} -x "**/node_modules**" "**/__pycache__/**" "**/.yarn/cache/**" "**/build/**" "**/.build/**" "**/dist/**" "**/.gradle/**" "**/bin/**" "**/vendor/**" "**/target/**" "**/.dart_tool/**"
+        run: zip -r -y clients-${{ matrix.client.language }}.zip ${{ matrix.client.path }} ${{ matrix.client.testsToStore }} ${{ matrix.client.guidesToStore }} -x "**/node_modules**" "**/__pycache__/**" "**/.yarn/cache/**" "**/build/**" "**/.build/**" "**/dist/**" "**/.gradle/**" "**/bin/**" "**/vendor/**" "**/target/**" "**/.dart_tool/**"
 
       - name: Store ${{ matrix.client.language }} clients
         if: ${{ matrix.client.isMainVersion }}

.github/workflows/pr-title.yml

@@ -16,4 +16,4 @@ jobs:
       - name: Pull Request title rules
         uses: Slashgear/[email protected]
         with:
-          regexp: '^(docs|chore|snippets)|((?:feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)\((?:clients|generators|playground|csharp|dart|go|java|javascript|kotlin|php|python|ruby|scala|swift|cts|specs|scripts|ci|templates|deps)\)): .+'
+          regexp: '^(docs|chore|snippets|guides)|((?:feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)\((?:clients|generators|playground|csharp|dart|go|java|javascript|kotlin|php|python|ruby|scala|swift|cts|specs|scripts|ci|templates|deps)\)): .+'

.gitignore

@@ -55,3 +55,5 @@ pubspec.lock
 swiftformat
 
 foo
+
+/snippets

config/generation.config.mjs

@@ -4,8 +4,7 @@ export const patterns = [
   'specs/bundled/*.json',
 
   'clients/**',
-  'snippets/**',
-  'snippets/guides/**',
+  'guides/**',
   '!clients/README.md',
   '!clients/**/.openapi-generator-ignore',
   'clients/**/.github/**',
@@ -23,13 +22,8 @@ export const patterns = [
   '!clients/algoliasearch-client-csharp/algoliasearch/Models/Common/**',
 
   'tests/output/csharp/global.json',
-  '!snippets/csharp/**',
-  'snippets/csharp/src/**.cs',
-  '!snippets/csharp/src/Program.cs',
 
   // Dart
-  '!snippets/dart/**',
-  'snippets/dart/lib/**',
   '!clients/algoliasearch-client-dart/**',
   'clients/algoliasearch-client-dart/packages/*/pubspec.yaml',
   'clients/algoliasearch-client-dart/packages/*/lib/*.dart',
@@ -53,7 +47,6 @@ export const patterns = [
   '!clients/algoliasearch-client-go/algolia/utils/*',
 
   '!tests/output/go/go.*',
-  '!snippets/go/go.*',
 
   // Java
   '!clients/algoliasearch-client-java/**',
@@ -63,10 +56,8 @@ export const patterns = [
   'clients/algoliasearch-client-java/algoliasearch/src/main/java/com/algolia/BuildConfig.java',
 
   'tests/output/java/build.gradle',
-  '!snippets/java/settings.gradle',
 
   // JavaScript
-  '!snippets/javascript/*.json',
   '!clients/algoliasearch-client-javascript/*',
   '!clients/algoliasearch-client-javascript/.yarn/**',
   '!clients/algoliasearch-client-javascript/scripts/**',
@@ -84,8 +75,6 @@ export const patterns = [
   'tests/output/javascript/package.json',
 
   // Kotlin
-  '!snippets/kotlin/**',
-  'snippets/kotlin/src/**',
   '!clients/algoliasearch-client-kotlin/**',
   'clients/algoliasearch-client-kotlin/gradle.properties',
   'clients/algoliasearch-client-kotlin/client/README.md',
@@ -106,7 +95,6 @@ export const patterns = [
   'clients/algoliasearch-client-php/composer.json',
 
   // Python
-  '!snippets/python/pyproject.toml',
   'clients/algoliasearch-client-python/**',
   '!clients/algoliasearch-client-python/algoliasearch/http/**',
   '!clients/algoliasearch-client-python/algoliasearch/py.typed',
@@ -121,7 +109,6 @@ export const patterns = [
   '!tests/output/python/**/__init__.py',
 
   // Ruby
-  '!snippets/ruby/Gemfile',
   '!clients/algoliasearch-client-ruby/**',
   'clients/algoliasearch-client-ruby/Gemfile.lock',
   'clients/algoliasearch-client-ruby/lib/algolia/**',
@@ -136,8 +123,6 @@ export const patterns = [
   'tests/output/ruby/Gemfile.lock',
 
   // Scala
-  '!snippets/scala/**',
-  'snippets/scala/src/**',
   '!clients/algoliasearch-client-scala/**',
   'clients/algoliasearch-client-scala/version.sbt',
   'clients/algoliasearch-client-scala/src/main/scala/algoliasearch/**',
@@ -148,9 +133,6 @@ export const patterns = [
   '!clients/algoliasearch-client-scala/src/main/scala/algoliasearch/extension/**',
 
   // Swift
-  '!snippets/swift/**',
-  'snippets/swift/Package.swift',
-  'snippets/swift/Sources/**',
   'clients/algoliasearch-client-swift/**',
   '!clients/algoliasearch-client-swift/*',
   'clients/algoliasearch-client-swift/AlgoliaSearchClient.podspec',

generators/src/main/java/com/algolia/codegen/cts/AlgoliaCTSGenerator.java

@@ -1,5 +1,6 @@
 package com.algolia.codegen.cts;
 
+import com.algolia.codegen.cts.guides.GuidesGenerator;
 import com.algolia.codegen.cts.lambda.*;
 import com.algolia.codegen.cts.manager.CTSManager;
 import com.algolia.codegen.cts.manager.CTSManagerFactory;
@@ -21,8 +22,7 @@ public class AlgoliaCTSGenerator extends DefaultCodegen {
 
   // cache the models
   private final Map<String, CodegenModel> models = new HashMap<>();
-  private String language;
-  private String client;
+  private String language, client, mode;
   private CTSManager ctsManager;
   private List<TestsGenerator> testsGenerators = new ArrayList<>();
 
@@ -42,7 +42,7 @@ public void processOpts() {
 
     language = (String) additionalProperties.get("language");
     client = (String) additionalProperties.get("client");
-    String mode = (String) additionalProperties.get("mode");
+    mode = (String) additionalProperties.get("mode");
     ctsManager = CTSManagerFactory.getManager(language, client);
 
     if (ctsManager == null) {
@@ -63,9 +63,13 @@ public void processOpts() {
       testsGenerators.add(new TestsClient(ctsManager, true));
       testsGenerators.add(new TestsClient(ctsManager, false));
     } else if (mode.equals("snippets")) {
-      ctsManager.addSnippetsSupportingFiles(supportingFiles);
+      ctsManager.addSnippetsSupportingFiles(supportingFiles, mode);
 
       testsGenerators.add(new SnippetsGenerator(ctsManager));
+    } else if (mode.equals("guides")) {
+      ctsManager.addSnippetsSupportingFiles(supportingFiles, mode);
+
+      testsGenerators.add(new GuidesGenerator(ctsManager));
     } else {
       throw new RuntimeException("Unknown mode: " + mode);
     }
@@ -142,6 +146,7 @@ public Map<String, Object> postProcessSupportingFileData(Map<String, Object> obj
       }
 
       // We can put whatever we want in the bundle, and it will be accessible in the template
+      bundle.put("mode", mode);
       bundle.put("client", Helpers.createClientName(importClientName, language) + "Client");
       bundle.put("clientPrefix", Helpers.createClientName(importClientName, language));
       bundle.put("hasRegionalHost", hasRegionalHost);
@@ -151,6 +156,7 @@ public Map<String, Object> postProcessSupportingFileData(Map<String, Object> obj
       bundle.put("isSyncClient", false);
       // special lambda for dynamic templates
       bundle.put("dynamicTemplate", new DynamicTemplateLambda(this));
+      bundle.put("dynamicSnippet", new DynamicSnippetLambda(this, models, operations, language, client));
       bundle.put("lambda", lambda);
 
       String languageVersion = ctsManager.getLanguageVersion((String) additionalProperties.getOrDefault("languageVersion", ""));

generators/src/main/java/com/algolia/codegen/cts/guides/GuidesGenerator.java

@@ -0,0 +1,55 @@
+package com.algolia.codegen.cts.guides;
+
+import com.algolia.codegen.cts.manager.CTSManager;
+import com.algolia.codegen.cts.tests.TestsGenerator;
+import com.algolia.codegen.utils.*;
+import java.io.File;
+import java.util.*;
+import org.openapitools.codegen.CodegenModel;
+import org.openapitools.codegen.CodegenOperation;
+import org.openapitools.codegen.SupportingFile;
+
+public class GuidesGenerator extends TestsGenerator {
+
+  public GuidesGenerator(CTSManager ctsManager) {
+    super(ctsManager);
+  }
+
+  @Override
+  public boolean available() {
+    File templates = new File("templates/" + language + "/guides/" + client);
+    return templates.exists();
+  }
+
+  @Override
+  public void addSupportingFiles(List<SupportingFile> supportingFiles, String outputFolder, String extension) {
+    if (!available()) {
+      return;
+    }
+
+    // same extension as the snippets
+    extension = Helpers.getClientConfigField(language, "snippets", "extension");
+    outputFolder = Helpers.getClientConfigField(language, "snippets", "outputFolder");
+
+    if (!outputFolder.equals("")) {
+      outputFolder = "/" + outputFolder + "/";
+    } else {
+      outputFolder = "/";
+    }
+
+    File templates = new File("templates/" + language + "/guides/" + client);
+    for (File f : templates.listFiles()) {
+      supportingFiles.add(
+        new SupportingFile(
+          "guides/" + client + "/" + f.getName(),
+          "guides/" + language + outputFolder + f.getName().replace(".mustache", "") + extension
+        )
+      );
+    }
+  }
+
+  @Override
+  public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation> operations, Map<String, Object> bundle) throws Exception {
+    // nothing to do here, the mustache uses dynamicSnippets lambda
+  }
+}

generators/src/main/java/com/algolia/codegen/cts/lambda/DynamicSnippetLambda.java

@@ -0,0 +1,81 @@
+package com.algolia.codegen.cts.lambda;
+
+import com.algolia.codegen.cts.tests.ParametersWithDataType;
+import com.algolia.codegen.cts.tests.Snippet;
+import com.algolia.codegen.exceptions.CTSException;
+import com.algolia.codegen.utils.Helpers;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.*;
+import com.samskivert.mustache.Mustache;
+import com.samskivert.mustache.Template;
+import io.swagger.v3.core.util.Json;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Map;
+import org.openapitools.codegen.CodegenModel;
+import org.openapitools.codegen.CodegenOperation;
+import org.openapitools.codegen.DefaultCodegen;
+import org.openapitools.codegen.TemplateManager;
+import org.openapitools.codegen.api.TemplatePathLocator;
+import org.openapitools.codegen.api.TemplatingEngineAdapter;
+import org.openapitools.codegen.api.TemplatingExecutor;
+import org.openapitools.codegen.templating.CommonTemplateContentLocator;
+import org.openapitools.codegen.templating.GeneratorTemplateContentLocator;
+import org.openapitools.codegen.templating.TemplateManagerOptions;
+
+public class DynamicSnippetLambda implements Mustache.Lambda {
+
+  private final TemplatingExecutor executor;
+  private final TemplatingEngineAdapter adaptor;
+
+  private final ParametersWithDataType paramsType;
+  private final Map<String, CodegenOperation> operations;
+
+  private final Map<String, Snippet> snippets;
+
+  public DynamicSnippetLambda(
+    DefaultCodegen generator,
+    Map<String, CodegenModel> models,
+    Map<String, CodegenOperation> operations,
+    String language,
+    String client
+  ) {
+    this.operations = operations;
+    this.paramsType = new ParametersWithDataType(models, language, client, true);
+
+    JsonNode snippetsFile = Helpers.readJsonFile("tests/CTS/guides/search.json");
+    this.snippets = Json.mapper().convertValue(snippetsFile, new TypeReference<Map<String, Snippet>>() {});
+
+    // we can't access the default template manager, so we have to create our own
+    TemplateManager templateManager = new TemplateManager(
+      new TemplateManagerOptions(generator.isEnableMinimalUpdate(), generator.isSkipOverwrite()),
+      generator.getTemplatingEngine(),
+      new TemplatePathLocator[] { new GeneratorTemplateContentLocator(generator), new CommonTemplateContentLocator() }
+    );
+
+    this.executor = templateManager;
+    this.adaptor = generator.getTemplatingEngine();
+  }
+
+  @Override
+  public void execute(Template.Fragment fragment, Writer writer) throws IOException, CTSException {
+    String snippetName = fragment.execute();
+    Snippet snippet = snippets.get(snippetName);
+    if (snippet == null) {
+      throw new CTSException("Cannot find snippet: " + snippetName);
+    }
+
+    String operationId = snippet.method;
+
+    CodegenOperation operation = operations.get(operationId);
+    if (operation == null) {
+      throw new CTSException("Cannot find operation for method: " + operationId);
+    }
+
+    // set the method attributes
+    Map<String, Object> context = (Map<String, Object>) fragment.context();
+    snippet.addMethodCall(context, paramsType, operation);
+
+    writer.write(adaptor.compileTemplate(executor, context, "tests/method.mustache"));
+  }
+}

generators/src/main/java/com/algolia/codegen/cts/manager/CSharpCTSManager.java

@@ -26,6 +26,18 @@ public void addTestsSupportingFiles(List<SupportingFile> supportingFiles) {
     supportingFiles.add(new SupportingFile("globaljson.mustache", "tests/output/csharp", "global.json"));
   }
 
+  @Override
+  public void addSnippetsSupportingFiles(List<SupportingFile> supportingFiles, String output) {
+    supportingFiles.add(new SupportingFile("snippets/src.csproj.mustache", output + "/csharp/src/src.csproj"));
+    supportingFiles.add(new SupportingFile("snippets/.gitignore.mustache", output + "/csharp/.gitignore"));
+    supportingFiles.add(new SupportingFile("snippets/Algolia.mustache", output + "/csharp/Algolia.sln"));
+    supportingFiles.add(new SupportingFile("snippets/dotnet-tools.mustache", output + "/csharp/.config/dotnet-tools.json"));
+
+    if (output.equals("snippets")) {
+      supportingFiles.add(new SupportingFile("snippets/Program.mustache", output + "/csharp/src/Program.cs"));
+    }
+  }
+
   @Override
   public void addDataToBundle(Map<String, Object> bundle) throws GeneratorException {
     bundle.put("packageVersion", getVersion());