feat: introduce based generator logic (#42)

* feat: introduce based generator logic

* chore: actually write the file

* chore: fixed typos

* Update src/generators.mjs

Co-authored-by: flakey5 <[email protected]>

* chore: explicitly import type

* chore: better types for the generators

* chore: more type optimisations

* chore: minor typo

* chore: even more type coersion and use plain object instead of Map

* chore: type directly on the generator

* chore: fix typo

* Apply suggestions from code review

Co-authored-by: Caner Akdas <[email protected]>

* chore: code review

---------

Co-authored-by: flakey5 <[email protected]>
Co-authored-by: Caner Akdas <[email protected]>

Author: 3 people

src/generators.mjs

@@ -0,0 +1,68 @@
+'use strict';
+
+import availableGenerators from './generators/index.mjs';
+
+/**
+ * @typedef {import('./types.d.ts').ApiDocMetadataEntry} ApiDocMetadataEntry Local type alias for the API doc metadata entry
+ * @typedef {{ ast: import('./generators/types.d.ts').GeneratorMetadata<ApiDocMetadataEntry, ApiDocMetadataEntry>}} AstGenerator The AST "generator" is a facade for the AST tree and it isn't really a generator
+ * @typedef {import('./generators/types.d.ts').AvailableGenerators & AstGenerator} AllGenerators A complete set of the available generators, including the AST one
+ *
+ * This method creates a system that allows you to register generators
+ * and then execute them in a specific order, keeping track of the
+ * generation process, and handling errors that may occur from the
+ * execution of generating content.
+ *
+ * When the final generator is reached, the system will return the
+ * final generated content.
+ *
+ * Generators can output content that can be consumed by other generators;
+ * Generators can also write to files. These would usually be considered
+ * the final generators in the chain.
+ *
+ * @param {ApiDocMetadataEntry} input The parsed API doc metadata entries
+ */
+const createGenerator = input => {
+  /**
+   * We store all the registered generators to be processed
+   * within a Record, so we can access their results at any time whenever needed
+   * (we store the Promises of the generator outputs)
+   *
+   * @type {{ [K in keyof AllGenerators]: ReturnType<AllGenerators[K]['generate']> }}
+   */
+  const cachedGenerators = { ast: Promise.resolve(input) };
+
+  /**
+   * Runs the Generator engine with the provided top-level input and the given generator options
+   *
+   * @param {import('./generators/types.d.ts').GeneratorOptions} options The options for the generator runtime
+   */
+  const runGenerators = async ({ output, generators }) => {
+    // Note that this method is blocking, and will only execute one generator per-time
+    // but it ensures all dependencies are resolved, and that multiple bottom-level generators
+    // can reuse the already parsed content from the top-level/dependency generators
+    for (const generatorName of generators) {
+      const { dependsOn, generate } = availableGenerators[generatorName];
+
+      // If the generator dependency has not yet been resolved, we resolve
+      // the dependency first before running the current generator
+      if (dependsOn && dependsOn in cachedGenerators === false) {
+        await runGenerators({ output, generators: [dependsOn] });
+      }
+
+      // Ensures that the dependency output gets resolved before we run the current
+      // generator with its dependency output as the input
+      const dependencyOutput = await cachedGenerators[dependsOn];
+
+      // Adds the current generator execution Promise to the cache
+      cachedGenerators[generatorName] = generate(dependencyOutput, { output });
+    }
+
+    // Returns the value of the last generator of the current pipeline
+    // Note that dependencies will be awaited (as shown on line 48)
+    return cachedGenerators[generators[generators.length - 1]];
+  };
+
+  return { runGenerators };
+};
+
+export default createGenerator;

src/generators/index.mjs

@@ -0,0 +1,6 @@
+'use strict';
+
+import jsonSimple from './json-simple/index.mjs';
+import legacyHtml from './legacy-html/index.mjs';
+
+export default { jsonSimple, legacyHtml };

src/generators/json-simple/index.mjs

@@ -0,0 +1,39 @@
+'use strict';
+
+import { writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+/**
+ * This generator generates a simplified JSON version of the API docs and returns it as a string
+ * this is not meant to be used for the final API docs, but for debugging and testing purposes
+ *
+ * This generator is a top-level generator, and it takes the raw AST tree of the API doc files
+ * and returns a stringified JSON version of the API docs.
+ *
+ * @typedef {import('../../types.d.ts').ApiDocMetadataEntry[]} Input
+ *
+ * @type {import('../types.d.ts').GeneratorMetadata<Input, string>}
+ */
+export default {
+  name: 'jsonSimple',
+
+  version: '1.0.0',
+
+  description:
+    'Generates the simple JSON version of the API docs, and returns it as a string',
+
+  dependsOn: 'ast',
+
+  async generate(input, options) {
+    // This simply grabs all the different files and stringifies them
+    const stringifiedContent = JSON.stringify(input, null, 2);
+
+    // Writes all the API docs stringified content into one file
+    // Note: The full JSON generator in the future will create one JSON file per top-level API doc file
+    await writeFile(
+      join(options.output, 'api-docs.json'),
+      stringifiedContent,
+      'utf-8'
+    );
+  },
+};

src/generators/legacy-html/assets/.gitkeep

@@ -0,0 +1 @@
+!.gitignore

src/generators/legacy-html/index.mjs

@@ -0,0 +1,27 @@
+'use strict';
+
+/**
+ * This generator generates the legacy HTML pages of the legacy API docs
+ * for retro-compatibility and while we are implementing the new 'react' and 'html' generators.
+ *
+ * This generator is a top-level generator, and it takes the raw AST tree of the API doc files
+ * and generates the HTML files to the specified output directory from the configuration settings
+ *
+ * @typedef {import('../../types.d.ts').ApiDocMetadataEntry[]} Input
+ *
+ * @type {import('../types.d.ts').GeneratorMetadata<Input, void>}
+ */
+export default {
+  name: 'legacyHtml',
+
+  version: '1.0.0',
+
+  description:
+    'Generates the legacy version of the API docs in HTML, with the assets and styles included as files',
+
+  dependsOn: 'ast',
+
+  async generate() {
+    throw new Error('Not yet implemented');
+  },
+};

src/generators/types.d.ts

@@ -0,0 +1,61 @@
+import type availableGenerators from './index.mjs';
+
+// All available generators as an inferable type, to allow Generator interfaces
+// to be type complete and runtime friendly within `runGenerators`
+export type AvailableGenerators = typeof availableGenerators;
+
+// This is the runtime config passed to the API doc generators
+export interface GeneratorOptions {
+  // The path used to output generated files, this is to be considered
+  // the base path that any generator will use for generating files
+  // This parameter accepts globs but when passed to generators will contain
+  // the already resolved absolute path to the output folder
+  output: string;
+  // A list of generators to be used in the API doc generation process;
+  // This is considered a "sorted" list of generators, in the sense that
+  // if the last entry of this list contains a generated value, we will return
+  // the value of the last generator in the list, if any.
+  generators: (keyof AvailableGenerators)[];
+}
+
+export interface GeneratorMetadata<I extends any, O extends any> {
+  // The name of the Generator. Must match the Key in the AvailableGenerators
+  name: keyof AvailableGenerators;
+
+  version: string;
+
+  description: string;
+
+  /**
+   * The immediate generator that this generator depends on.
+   * For example, the `html` generator depends on the `react` generator.
+   *
+   * If a given generator has no "before" generator, it will be considered a top-level
+   * generator, and run in parallel.
+   *
+   * Assume you pass to the `createGenerator`: ['json', 'html'] as the generators,
+   * this means both the 'json' and the 'html' generators will be executed and generate their
+   * own outputs in parallel. If the 'html' generator depends on the 'react' generator, then
+   * the 'react' generator will be executed first, then the 'html' generator.
+   *
+   * But both 'json' and 'html' generators will be executed in parallel.
+   *
+   * If you pass `createGenerator` with ['react', 'html'], the 'react' generator will be executed first,
+   * as it is a top level generator and then the 'html' generator would be executed after the 'react' generator.
+   *
+   * The 'ast' generator is the top-level parser, and if 'ast' is passed to `dependsOn`, then the generator
+   * will be marked as a top-level generator.
+   */
+  dependsOn: keyof AvailableGenerators | 'ast';
+
+  /**
+   * Generators are abstract and the different generators have different sort of inputs and outputs.
+   * For example, a MDX generator would take the raw AST and output MDX with React Components;
+   * Whereas a JSON generator would take the raw AST and output JSON;
+   * Then a React generator could receive either the raw AST or the MDX output and output React Components.
+   * (depending if they support such I/O)
+   *
+   * Hence you can combine different generators to achieve different outputs.
+   */
+  generate: (input: I, options: Partial<GeneratorOptions>) => Promise<O>;
+}

src/metadata.mjs

@@ -1,6 +1,10 @@
 'use strict';
 
 /**
+ * @typedef {import('./types.d.ts').ApiDocMetadataEntry} ApiDocMetadataEntry Local type alias for the API doc metadata entry
+ * @typedef {import('./types.d.ts').ApiDocRawMetadataEntry} ApiDocRawMetadataEntry Local type alias for the API doc raw metadata entry
+ * @typedef {import('./types.d.ts').HeadingMetadataEntry} HeadingMetadataEntry Local type alias for the heading metadata entry
+ *
  * This method allows us to handle creation of Metadata entries
  * within the current scope of API docs being parsed
  *
@@ -15,9 +19,9 @@ const createMetadata = slugger => {
    * transformed into NavigationEntries and MetadataEntries
    *
    * @type {{
-   *  heading: import('./types.d.ts').HeadingMetadataEntry,
-   *  stability: import('./types.d.ts').ApiDocMetadataEntry['stability'],
-   *  properties: import('./types.d.ts').ApiDocRawMetadataEntry,
+   *  heading: HeadingMetadataEntry,
+   *  properties: ApiDocRawMetadataEntry,
+   *  stability: ApiDocMetadataEntry['stability'],
    * }}
    */
   const internalMetadata = {
@@ -27,23 +31,23 @@ const createMetadata = slugger => {
       name: undefined,
       depth: -1,
     },
-    stability: undefined,
     properties: {},
+    stability: undefined,
   };
 
   return {
     /**
      * Set the Heading of a given Metadata
      *
-     * @param {import('./types.d.ts').HeadingMetadataEntry} heading The new heading metadata
+     * @param {HeadingMetadataEntry} heading The new heading metadata
      */
     setHeading: heading => {
       internalMetadata.heading = heading;
     },
     /**
      * Set the Stability Index of a given Metadata
      *
-     * @param {import('./types.d.ts').ApiDocMetadataEntry['stability']} stability The new stability metadata
+     * @param {ApiDocMetadataEntry['stability']} stability The new stability metadata
      */
     setStability: stability => {
       internalMetadata.stability = stability;
@@ -57,7 +61,7 @@ const createMetadata = slugger => {
      * meaning that this method can be called multiple times to update the properties
      * and complement each set of data.
      *
-     * @param {Partial<import('./types.d.ts').ApiDocRawMetadataEntry>} properties Extra Metadata properties to be defined
+     * @param {Partial<ApiDocRawMetadataEntry>} properties Extra Metadata properties to be defined
      */
     updateProperties: properties => {
       internalMetadata.properties = {
@@ -74,8 +78,8 @@ const createMetadata = slugger => {
      * as it can be manipulated outside of the scope of the generation of the content
      *
      * @param {import('vfile').VFile} apiDoc The API doc file being parsed
-     * @param {import('./types.d.ts').ApiDocMetadataEntry['content']} section An AST tree containing the Nodes of the API doc entry section
-     * @returns {import('./types.d.ts').ApiDocMetadataEntry} The locally created Metadata entries
+     * @param {ApiDocMetadataEntry['content']} section An AST tree containing the Nodes of the API doc entry section
+     * @returns {ApiDocMetadataEntry} The locally created Metadata entries
      */
     create: (apiDoc, section) => {
       // This is the ID of a certain Navigation entry, which allows us to anchor

src/parser.mjs

@@ -111,7 +111,7 @@ const createParser = () => {
 
       // This is the cutover index of the subtree that we should get
       // of all the Nodes within the AST tree that belong to this section
-      // If `next` is equals the current heading, it means ther's no next heading
+      // If `next` is equals the current heading, it means there's no next heading
       // and we are reaching the end of the document, hence the cutover should be the end of
       // the document itself.
       const stop =

src/types.d.ts

@@ -2,7 +2,7 @@ import { Parent, Node } from 'unist';
 
 // String serialization of the AST tree
 // @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior
-export interface WithJSON<T extends Node, J = any> extends T {
+export interface WithJSON<T extends Node, J extends any = any> extends T {
   toJSON: () => J;
 }