docs: generate guides from snippets (#3139)

Co-authored-by: Pierre Millot <[email protected]>
Co-authored-by: algolia-bot <[email protected]>

Author: 3 people

.gitignore

@@ -34,6 +34,7 @@ generators/bin
 
 **/java/bin/
 **/kotlin/bin/
+.kotlin/
 
 # Dart
 .dart_tool/

generators/src/main/java/com/algolia/codegen/cts/AlgoliaCTSGenerator.java

@@ -60,7 +60,7 @@ public void processOpts() {
     if (mode.equals("tests")) {
       ctsManager.addTestsSupportingFiles(supportingFiles);
 
-      testsGenerators.add(new TestsRequest(language, client));
+      testsGenerators.add(new TestsRequest(language, client, false));
       testsGenerators.add(new TestsClient(language, client));
     } else if (mode.equals("snippets")) {
       ctsManager.addSnippetsSupportingFiles(supportingFiles);

generators/src/main/java/com/algolia/codegen/cts/snippets/SnippetsGenerator.java

@@ -11,7 +11,7 @@
 public class SnippetsGenerator extends TestsRequest {
 
   public SnippetsGenerator(String language, String client) {
-    super(language, client);
+    super(language, client, true);
   }
 
   @Override

generators/src/main/java/com/algolia/codegen/cts/tests/Request.java

@@ -12,6 +12,7 @@
 public class Request {
 
   public String testName;
+  public boolean isSnippet;
 
   public Map<String, Object> parameters;
   public RequestOptions requestOptions;
@@ -23,6 +24,7 @@ public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append("class Request {\n");
     sb.append("    testName: ").append(testName).append("\n");
+    sb.append("    isSnippet").append(isSnippet).append("\n");
     sb.append("    parameters: ").append(parameters).append("\n");
     sb.append("    requestOptions: ").append(requestOptions).append("\n");
     sb.append("    request: ").append(request).append("\n");

generators/src/main/java/com/algolia/codegen/cts/tests/TestsRequest.java

@@ -13,8 +13,11 @@
 
 public class TestsRequest extends TestsGenerator {
 
-  public TestsRequest(String language, String client) {
+  private final boolean withSnippets;
+
+  public TestsRequest(String language, String client, boolean withSnippets) {
     super(language, client);
+    this.withSnippets = withSnippets;
   }
 
   protected Map<String, Request[]> loadRequestCTS() throws Exception {
@@ -84,13 +87,14 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
       }
       Request[] op = cts.get(operationId);
 
-      List<Object> tests = new ArrayList<>();
+      List<Map<String, Object>> tests = new ArrayList<>();
       for (int i = 0; i < op.length; i++) {
         Map<String, Object> test = new HashMap<>();
         Request req = op[i];
         test.put("method", operationId);
-        test.put("testName", req.testName == null ? operationId + i : req.testName);
-        test.put("testIndex", i);
+        test.put("testName", req.testName == null ? operationId : req.testName);
+        test.put("testIndex", i == 0 ? "" : i);
+        test.put("isSnippet", req.isSnippet);
         if (ope.returnType != null && ope.returnType.length() > 0) {
           test.put("returnType", camelize(ope.returnType));
         }
@@ -174,9 +178,20 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
       testObj.put("tests", tests);
       testObj.put("operationId", operationId);
 
-      Map<String, Object> snippet = (Map<String, Object>) tests.get(0);
-      snippet.put("description", snippet.get("testName"));
-      testObj.put("snippet", snippet);
+      if (withSnippets) {
+        List<Map<String, Object>> snippets = tests.stream().filter(t -> (boolean) t.getOrDefault("isSnippet", false)).toList();
+        if (snippets.size() == 0) {
+          Map<String, Object> snippet = tests.get(0);
+          snippet.put("description", snippet.get("testName"));
+          snippet.put("testName", "default");
+          snippets = List.of(snippet);
+        } else {
+          for (Map<String, Object> snippet : snippets) {
+            snippet.put("description", snippet.get("testName"));
+          }
+        }
+        testObj.put("snippets", snippets);
+      }
 
       blocks.add(testObj);
     }

scripts/.eslintrc.cjs

@@ -41,5 +41,7 @@ module.exports = {
       },
     ],
     '@typescript-eslint/sort-type-union-intersection-members': 0,
+    'no-param-reassign': 0,
+    '@typescript-eslint/consistent-type-assertions': 0,
   },
 };

scripts/buildClients.ts

@@ -53,7 +53,6 @@ export async function buildClients(generators: Generator[]): Promise<void> {
   const generatorsMap = generators.reduce(
     (map, gen) => {
       if (!(gen.language in map)) {
-        // eslint-disable-next-line no-param-reassign
         map[gen.language] = [];
       }
 

scripts/buildSpecs.ts

@@ -5,7 +5,7 @@ import yaml from 'js-yaml';
 import { Cache } from './cache.js';
 import { GENERATORS, capitalize, createClientName, exists, run, toAbsolutePath } from './common.js';
 import { createSpinner } from './spinners.js';
-import type { CodeSamples, Language, SnippetSamples, Spec } from './types.js';
+import type { CodeSamples, Language, SnippetForMethod, SnippetSamples, Spec } from './types.js';
 
 const ALGOLIASEARCH_LITE_OPERATIONS = ['search', 'customPost'];
 
@@ -22,6 +22,38 @@ function getCodeSampleLabel(language: Language): CodeSamples['label'] {
   }
 }
 
+// Iterates over the snippet samples and sanitize the data to only keep the method part in order to use it in the guides.
+function transformCodeSamplesToGuideMethods(snippetSamples: SnippetSamples): string {
+  for (const [language, operationWithSample] of Object.entries(snippetSamples)) {
+    for (const [operation, samples] of Object.entries(operationWithSample)) {
+      if (operation === 'import') {
+        continue;
+      }
+
+      for (const [sampleName, sample] of Object.entries(samples)) {
+        const sampleMatch = sample.match(
+          /.*Initialize the client\n(.*)((.|\n)*)(.*Call the API\n)((.|\n)*)/,
+        );
+        if (!sampleMatch) {
+          continue;
+        }
+
+        const initLine = sampleMatch[1];
+        const callLine = sampleMatch[5];
+
+        if (!('init' in snippetSamples[language])) {
+          snippetSamples[language].init = {
+            default: initLine.replace(/\n$/, ''),
+          };
+        }
+        snippetSamples[language][operation][sampleName] = callLine.replace(/\n$/, '');
+      }
+    }
+  }
+
+  return JSON.stringify(snippetSamples, null, 2);
+}
+
 // For a given `clientName`, reads the matching snippet file for every available clients and builds an hashmap of snippets per operationId per language.
 async function transformSnippetsToCodeSamples(clientName: string): Promise<SnippetSamples> {
   const snippetSamples = Object.values(GENERATORS).reduce(
@@ -42,26 +74,40 @@ async function transformSnippetsToCodeSamples(clientName: string): Promise<Snipp
       'utf8',
     );
 
+    const importMatch = snippetFileContent.match(/>IMPORT\n([\s\S]*?)\n.*IMPORT</);
+    if (importMatch) {
+      snippetSamples[gen.language].import = {
+        default: importMatch[1].replace(/\n$/, ''),
+      };
+    }
+
     // iterate over every matches (operationId) and store it in the hashmap for later use
-    for (const match of snippetFileContent.matchAll(/>SEPARATOR (.+)\n([\s\S]*?)SEPARATOR</g)) {
+    for (const match of snippetFileContent.matchAll(
+      />SEPARATOR (\w+) (.*)\n([\s\S]*?)SEPARATOR</g,
+    )) {
       const lines: string[] = match[0].split('\n').slice(1, -1);
       if (!lines.length) {
         throw new Error(`No snippet found for ${gen.language} ${gen.client}`);
       }
 
-      if (!snippetSamples[gen.language]) {
-        snippetSamples[gen.language] = {};
+      const operationId = match[1];
+      const testName = match[2] || 'default';
+
+      if (!snippetSamples[gen.language][operationId]) {
+        snippetSamples[gen.language][operationId] = {};
       }
 
-      snippetSamples[gen.language][match[1]] = '';
+      const snippetForMethod: SnippetForMethod = snippetSamples[gen.language][operationId];
+
+      snippetForMethod[testName] = '';
 
       const indent = lines[0].length - lines[0].trim().length;
       // skip first and last lines because they contain the SEPARATOR or operationId
       lines.forEach((line) => {
         // best effort to determine how far the snippet is indented so we
         // can have every snippets in the documentation on the far left
         // without impacting the formatting
-        snippetSamples[gen.language][match[1]] += `${line.slice(indent).replaceAll(/\t/g, '  ')}\n`;
+        snippetForMethod[testName] += `${line.slice(indent).replaceAll(/\t/g, '  ')}\n`;
       });
     }
   }
@@ -94,7 +140,16 @@ async function transformBundle({
 
   const bundledSpec = yaml.load(await fsp.readFile(bundledPath, 'utf8')) as Spec;
   const tagsDefinitions = bundledSpec.tags;
-  const snippetSamples = docs ? await transformSnippetsToCodeSamples(clientName) : {};
+  const snippetSamples = docs
+    ? await transformSnippetsToCodeSamples(clientName)
+    : ({} as SnippetSamples);
+
+  if (docs) {
+    await fsp.writeFile(
+      toAbsolutePath(`website/src/generated/${clientName}-snippets.js`),
+      `export const snippets = ${transformCodeSamplesToGuideMethods(JSON.parse(JSON.stringify(snippetSamples)))}`,
+    );
+  }
 
   for (const [pathKey, pathMethods] of Object.entries(bundledSpec.paths)) {
     for (const [method, specMethod] of Object.entries(pathMethods)) {
@@ -120,11 +175,13 @@ async function transformBundle({
           specMethod['x-codeSamples'] = [];
         }
 
-        specMethod['x-codeSamples'].push({
-          lang: gen.language,
-          label: getCodeSampleLabel(gen.language),
-          source: snippetSamples[gen.language][specMethod.operationId],
-        });
+        if (snippetSamples[gen.language][specMethod.operationId]) {
+          specMethod['x-codeSamples'].push({
+            lang: gen.language,
+            label: getCodeSampleLabel(gen.language),
+            source: Object.values(snippetSamples[gen.language][specMethod.operationId])[0],
+          });
+        }
       }
 
       if (!bundledSpec.paths[pathKey][method].tags) {

scripts/ci/githubActions/setRunVariables.ts

@@ -56,7 +56,6 @@ export const DEPENDENCIES = LANGUAGES.reduce(
     const key = `${lang.toUpperCase()}_CLIENT_CHANGED`;
     const langFolder = getLanguageFolder(lang);
 
-    // eslint-disable-next-line no-param-reassign
     finalDependencies[key] = [
       ':!**node_modules',
       `templates/${lang}`,

scripts/common.ts

@@ -47,7 +47,6 @@ export const GENERATORS = Object.entries(clientsConfig).reduce(
         clientName = client;
       }
 
-      // eslint-disable-next-line no-param-reassign
       current[key] = {
         additionalProperties: {},
         ...gen,
@@ -59,7 +58,6 @@ export const GENERATORS = Object.entries(clientsConfig).reduce(
 
       // guess the package name for js from the output folder variable
       if (language === 'javascript') {
-        // eslint-disable-next-line no-param-reassign
         current[key].additionalProperties.packageName = output.substring(
           output.lastIndexOf('/') + 1,
         );

scripts/types.ts

@@ -124,7 +124,9 @@ export type CodeSamples = {
   source: string;
 };
 
-export type SnippetSamples = Record<Language, Record<string, string>>;
+export type SnippetForMethod = Record<string, string>;
+export type SnippetForLanguage = Record<string, SnippetForMethod>;
+export type SnippetSamples = Record<Language, SnippetForLanguage>;
 
 /**
  * Paths of a spec.

scripts/website/build.sh

@@ -4,7 +4,6 @@
 touch website/yarn.lock
 
 # Generate the file with constants
-mkdir -p website/src/generated
 cat config/clients.config.json | jq -r '"export const versions = \(map_values(.packageVersion))"' > website/src/generated/variables.js
 
 # install website deps and build

snippets/kotlin/build.gradle.kts

@@ -27,6 +27,7 @@ configure<SpotlessExtension> {
                     "ktlint_standard_no-wildcard-imports" to "disabled",
                     "ktlint_standard_trailing-comma-on-declaration-site" to "disabled",
                     "ktlint_standard_filename" to "disabled",
+                    "ktlint_standard_import-ordering" to "disabled",
                 ),
             )
     }

snippets/ruby/Gemfile.lock

@@ -12,7 +12,7 @@ GEM
     ast (2.4.2)
     connection_pool (2.4.1)
     dotenv (3.1.2)
-    faraday (2.9.0)
+    faraday (2.9.1)
       faraday-net_http (>= 2.0, < 3.2)
     faraday-net_http (3.1.0)
       net-http
@@ -26,15 +26,16 @@ GEM
     net-http-persistent (4.0.2)
       connection_pool (~> 2.2)
     parallel (1.24.0)
-    parser (3.3.1.0)
+    parser (3.3.2.0)
       ast (~> 2.4.1)
       racc
-    racc (1.7.3)
+    racc (1.8.0)
     rainbow (3.1.1)
     rake (13.2.1)
-    regexp_parser (2.9.1)
+    regexp_parser (2.9.2)
     rexml (3.2.8)
-    rubocop (1.63.5)
+      strscan (>= 3.0.9)
+    rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -48,10 +49,12 @@ GEM
     rubocop-ast (1.31.3)
       parser (>= 3.3.1.0)
     ruby-progressbar (1.13.0)
+    strscan (3.1.0)
     unicode-display_width (2.5.0)
     uri (0.13.0)
 
 PLATFORMS
+  aarch64-linux
   x86_64-linux
 
 DEPENDENCIES

specs/common/parameters.yml

@@ -63,6 +63,7 @@ EndDateRequired:
 ForwardToReplicas:
   in: query
   name: forwardToReplicas
+  required: false
   description: Whether changes are applied to replica indices.
   schema:
     type: boolean
@@ -73,6 +74,7 @@ Page:
   description: |
     Requested page of the API response.
     If `null`, the API response is not paginated.
+  required: false
   schema:
     oneOf:
       - type: integer
@@ -97,6 +99,7 @@ HitsPerPage:
   in: query
   name: hitsPerPage
   description: Number of hits per page.
+  required: false
   schema:
     type: integer
     default: 100
@@ -105,6 +108,7 @@ Offset:
   in: query
   name: offset
   description: Position of the first item to return.
+  required: false
   schema:
     type: integer
     default: 0
@@ -114,6 +118,7 @@ Limit:
   in: query
   name: limit
   description: Number of items to return.
+  required: false
   schema:
     type: integer
     default: 10