feat apilinks.json generator

Closes #152

Signed-off-by: flakey5 <[email protected]>

Author: flakey5

README.md

@@ -39,6 +39,6 @@ Options:
   -o, --output <path>        Specify the relative or absolute output directory
   -v, --version <semver>     Specify the target version of Node.js, semver compliant (default: "v22.6.0")
   -c, --changelog <url>      Specify the path (file: or https://) to the CHANGELOG.md file (default: "https://raw.githubusercontent.com/nodejs/node/HEAD/CHANGELOG.md")
-  -t, --target [mode...]     Set the processing target modes (choices: "json-simple", "legacy-html", "legacy-html-all", "man-page", "legacy-json", "legacy-json-all", "addon-verify")
+  -t, --target [mode...]     Set the processing target modes (choices: "json-simple", "legacy-html", "legacy-html-all", "man-page", "legacy-json", "legacy-json-all", "addon-verify", "api-links")
   -h, --help                 display help for command
 ```

bin/cli.mjs

@@ -68,14 +68,22 @@ program
  */
 const { input, output, target = [], version, changelog } = program.opts();
 
-const { loadFiles } = createLoader();
-const { parseApiDocs } = createParser();
+const { loadFiles, loadJsFiles } = createLoader();
+const { parseApiDocs, parseJsSources } = createParser();
 
 const apiDocFiles = loadFiles(input);
 
 const parsedApiDocs = await parseApiDocs(apiDocFiles);
 
-const { runGenerators } = createGenerator(parsedApiDocs);
+const sourceFiles = loadJsFiles(
+  parsedApiDocs
+    .map(apiDoc => apiDoc.source_link_local)
+    .filter(path => path !== undefined && path.endsWith('.js'))
+);
+
+const parsedJsFiles = await parseJsSources(sourceFiles);
+
+const { runGenerators } = createGenerator(parsedApiDocs, parsedJsFiles);
 
 // Retrieves Node.js release metadata from a given Node.js version and CHANGELOG.md file
 const { getAllMajors } = createNodeReleases(changelog);

package-lock.json

@@ -29,6 +29,7 @@
     "prettier": "3.4.2"
   },
   "dependencies": {
+    "acorn": "^8.14.0",
     "commander": "^13.0.0",
     "github-slugger": "^2.0.0",
     "glob": "^11.0.0",

package.json

@@ -19,16 +19,20 @@ import availableGenerators from './generators/index.mjs';
  * the final generators in the chain.
  *
  * @param {ApiDocMetadataEntry} input The parsed API doc metadata entries
+ * @param {Array<import('acorn').Program>} parsedJsFiles
  */
-const createGenerator = input => {
+const createGenerator = (input, parsedJsFiles) => {
   /**
    * We store all the registered generators to be processed
    * within a Record, so we can access their results at any time whenever needed
    * (we store the Promises of the generator outputs)
    *
    * @type {{ [K in keyof AllGenerators]: ReturnType<AllGenerators[K]['generate']> }}
    */
-  const cachedGenerators = { ast: Promise.resolve(input) };
+  const cachedGenerators = {
+    ast: Promise.resolve(input),
+    'ast-js': Promise.resolve(parsedJsFiles),
+  };
 
   /**
    * Runs the Generator engine with the provided top-level input and the given generator options

src/generators.mjs

@@ -0,0 +1,95 @@
+'use strict';
+
+import { basename, dirname, join } from 'node:path';
+import { writeFile } from 'node:fs/promises';
+import { getGitRepository, getGitTag } from '../../utils/git.mjs';
+import { extractExports } from './utils/extractExports.mjs';
+import { findDefinitions } from './utils/findDefinitions.mjs';
+
+/**
+ * This generator is responsible for mapping publicly accessible functions in
+ * Node.js to their source locations in the Node.js repository.
+ *
+ * This is a top-level generator. It takes in the raw AST tree of the JavaScript
+ * source files. It outputs a `apilinks.json` file into the specified output
+ * directory.
+ *
+ * @typedef {Array<JsProgram>} Input
+ *
+ * @type {import('../types.d.ts').GeneratorMetadata<Input, Record<string, string>>}
+ */
+export default {
+  name: 'api-links',
+
+  version: '1.0.0',
+
+  description:
+    'Creates a mapping of publicly accessible functions to their source locations in the Node.js repository.',
+
+  dependsOn: 'ast-js',
+
+  /**
+   * Generates the `apilinks.json` file.
+   *
+   * @param {Input} input
+   * @param {Partial<GeneratorOptions>} options
+   */
+  async generate(input, { output }) {
+    /**
+     * @type {Record<string, string>}
+     */
+    const definitions = {};
+
+    /**
+     * @type {string}
+     */
+    let baseGithubLink;
+
+    input.forEach(program => {
+      /**
+       * Mapping of definitions to their line number
+       * @type {Record<string, number>}
+       * @example { 'someclass.foo', 10 }
+       */
+      const nameToLineNumberMap = {};
+
+      const programBasename = basename(program.path, '.js');
+
+      const exports = extractExports(
+        program,
+        programBasename,
+        nameToLineNumberMap
+      );
+
+      findDefinitions(program, programBasename, nameToLineNumberMap, exports);
+
+      if (!baseGithubLink) {
+        const directory = dirname(program.path);
+
+        const repository = getGitRepository(directory);
+
+        const tag = getGitTag(directory);
+
+        baseGithubLink = `https://github.com/${repository}/blob/${tag}`;
+      }
+
+      const githubLink =
+        `${baseGithubLink}/lib/${programBasename}.js`.replaceAll('\\', '/');
+
+      Object.keys(nameToLineNumberMap).forEach(key => {
+        const lineNumber = nameToLineNumberMap[key];
+
+        definitions[key] = `${githubLink}#L${lineNumber}`;
+      });
+    });
+
+    if (output) {
+      await writeFile(
+        join(output, 'apilinks.json'),
+        JSON.stringify(definitions)
+      );
+    }
+
+    return definitions;
+  },
+};

src/generators/api-links/index.mjs

@@ -0,0 +1,4 @@
+export interface ProgramExports {
+  ctors: Array<string>;
+  identifiers: Array<string>;
+}

src/generators/api-links/types.d.ts

@@ -0,0 +1,215 @@
+// @ts-check
+'use strict';
+
+/**
+ * @param {import('acorn').AssignmentExpression} expression
+ * @param {import('acorn').SourceLocation} loc
+ * @param {string} basename
+ * @param {Record<string, number>} nameToLineNumberMap
+ * @returns {import('../types').ProgramExports | undefined}
+ */
+function extractExpression(expression, loc, basename, nameToLineNumberMap) {
+  /**
+   * @example `a=b`, lhs=`a` and rhs=`b`
+   */
+  let { left: lhs, right: rhs } = expression;
+
+  if (lhs.type !== 'MemberExpression') {
+    return undefined;
+  }
+
+  if (lhs.object.type === 'MemberExpression') {
+    lhs = lhs.object;
+  }
+
+  /**
+   * @type {import('../types').ProgramExports}
+   */
+  const exports = {
+    ctors: [],
+    identifiers: [],
+  };
+
+  if (lhs.object.name === 'exports') {
+    // Assigning a property in `module.exports` (i.e. `module.exports.asd = ...`)
+    const { name } = lhs.property;
+
+    switch (rhs.type) {
+      case 'FunctionExpression':
+        // module.exports.something = () => {}
+        nameToLineNumberMap[`${basename}.${name}`] = loc.start.line;
+        break;
+
+      case 'Identifier':
+        // module.exports.asd = something
+        // TODO indirects?
+        console.log('indir', name);
+        break;
+
+      default:
+        exports.identifiers.push(name);
+        break;
+    }
+  } else if (lhs.object.name === 'module' && lhs.property.name === 'exports') {
+    // Assigning `module.exports` as a whole, (i.e. `module.exports = {}`)
+    while (rhs.type === 'AssignmentExpression') {
+      // Move right until we find the value of the assignment
+      //  (i.e. `a=b`, we want `b`).
+      rhs = rhs.right;
+    }
+
+    switch (rhs.type) {
+      case 'NewExpression':
+        // module.exports = new Asd()
+        exports.ctors.push(rhs.callee.name);
+        break;
+
+      case 'ObjectExpression':
+        // module.exports = {}
+        // We need to go through all of the properties and add register them
+        rhs.properties.forEach(({ value }) => {
+          if (value.type !== 'Identifier') {
+            return;
+          }
+
+          exports.identifiers.push(value.name);
+
+          if (/^[A-Z]/.test(value.name[0])) {
+            exports.ctors.push(value.name);
+          }
+        });
+
+        break;
+
+      default:
+        exports.identifiers.push(rhs.name);
+        break;
+    }
+  }
+
+  return exports;
+}
+
+/**
+ * @param {import('acorn').VariableDeclarator} declaration
+ * @param {import('acorn').SourceLocation} loc
+ * @param {string} basename
+ * @param {Record<string, number>} nameToLineNumberMap
+ * @returns {import('../types').ProgramExports | undefined}
+ */
+function extractVariableDeclaration(
+  { id, init },
+  loc,
+  basename,
+  nameToLineNumberMap
+) {
+  while (init && init.type === 'AssignmentExpression') {
+    // Move left until we get to what we're assigning to
+    //  (i.e. `a=b`, we want `a`)
+    init = init.left;
+  }
+
+  if (!init || init.type !== 'MemberExpression') {
+    // Doesn't exist or we're not writing to a member (probably a normal var,
+    //  like `const a = 123`)
+    return undefined;
+  }
+
+  /**
+   * @type {import('../types').ProgramExports}
+   */
+  const exports = {
+    ctors: [],
+    identifiers: [],
+  };
+
+  if (init.object.name === 'exports') {
+    // Assigning a property in `module.exports` (i.e. `module.exports.asd = ...`)
+    nameToLineNumberMap[`${basename}.${init.property.name}`] = loc.start.line;
+  } else if (
+    init.object.name === 'module' &&
+    init.property.name === 'exports'
+  ) {
+    // Assigning `module.exports` as a whole, (i.e. `module.exports = {}`)
+    exports.ctors.push(id.name);
+    nameToLineNumberMap[id.name] = loc.start.line;
+  }
+
+  return exports;
+}
+
+/**
+ * We need to find what a source file exports so we know what to include in
+ * the final result. We can do this by going through every statement in the
+ * program looking for assignments to `module.exports`.
+ *
+ * Noteworthy that exports can happen throughout the program so we need to
+ * go through the entire thing.
+ *
+ * @param {import('acorn').Program} program
+ * @param {string} basename
+ * @param {Record<string, number>} nameToLineNumberMap
+ * @returns {import('../types').ProgramExports}
+ */
+export function extractExports(program, basename, nameToLineNumberMap) {
+  /**
+   * @type {import('../types').ProgramExports}
+   */
+  const exports = {
+    ctors: [],
+    identifiers: [],
+  };
+
+  program.body.forEach(statement => {
+    const { loc } = statement;
+    if (!loc) {
+      return;
+    }
+
+    switch (statement.type) {
+      case 'ExpressionStatement': {
+        const { expression } = statement;
+        if (expression.type !== 'AssignmentExpression' || !loc) {
+          break;
+        }
+
+        const expressionExports = extractExpression(
+          expression,
+          loc,
+          basename,
+          nameToLineNumberMap
+        );
+
+        if (expressionExports) {
+          exports.ctors.push(...expressionExports.ctors);
+          exports.identifiers.push(...expressionExports.identifiers);
+        }
+
+        break;
+      }
+
+      case 'VariableDeclaration': {
+        statement.declarations.forEach(declaration => {
+          const variableExports = extractVariableDeclaration(
+            declaration,
+            loc,
+            basename,
+            nameToLineNumberMap
+          );
+
+          if (variableExports) {
+            exports.ctors.push(...variableExports.ctors);
+            exports.identifiers.push(...variableExports.identifiers);
+          }
+        });
+
+        break;
+      }
+
+      default:
+        break;
+    }
+  });
+
+  return exports;
+}