ci(NODE-5403): add string deserialization benchmark to ci (#593)

Author: W-A-James

.evergreen/config.yml

@@ -41,24 +41,25 @@ functions:
           - .evergreen/install-dependencies.sh
 
   run tests:
-    - command: shell.exec
+    - command: subprocess.exec
       type: test
       params:
         working_dir: src
-        script: |
-          ${PREPARE_SHELL}
-          echo "NO_BIGINT=${NO_BIGINT} TEST_TARGET=${TEST_TARGET}"
-          NO_BIGINT=${NO_BIGINT} ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh ${TEST_TARGET}
+        add_expansions_to_env: true
+        binary: bash
+        args:
+          - .evergreen/run-tests.sh
 
   run checks:
-    - command: shell.exec
+    - command: subprocess.exec
       type: test
       params:
         working_dir: src
-        script: |
-          ${PREPARE_SHELL}
-          echo "TEST_TARGET=${TEST_TARGET}"
-          bash ${PROJECT_DIRECTORY}/.evergreen/run-checks.sh
+        add_expansions_to_env: true
+        binary: bash
+        args:
+          - .evergreen/run-checks.sh
+
   run typescript:
     - command: subprocess.exec
       type: test
@@ -93,7 +94,15 @@ functions:
           PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
         args:
           - .evergreen/run-bundling-test.sh
-
+  run benchmarks:
+    - command: subprocess.exec
+      type: test
+      params:
+        working_dir: src
+        binary: bash
+        add_expansions_to_env: true
+        args:
+          - .evergreen/run-benchmarks.sh
 tasks:
   - name: node-tests-v16
     tags: ["node"]
@@ -200,6 +209,18 @@ tasks:
         vars:
           TS_VERSION: "next"
           TRY_COMPILING_LIBRARY: "false"
+  - name: run-benchmarks-node
+    commands:
+      - func: fetch source
+        vars:
+          NODE_LTS_VERSION: v18.16.0
+      - func: install dependencies
+      - func: run benchmarks
+        vars:
+          WEB: false 
+      - command: perf.send
+        params:
+          file: src/benchmarks.json
   - name: check-eslint-plugin
     commands:
       - func: fetch source
@@ -222,3 +243,8 @@ buildvariants:
       - check-typescript-current
       - check-typescript-next
       - bundling-tests
+  - name: perf
+    display_name: RHEL 9.0 perf
+    run_on: rhel90-dbx-perf-large
+    tasks:
+      - run-benchmarks-node

.evergreen/install-dependencies.sh

@@ -36,6 +36,7 @@ while IFS=$'\t' read -r -a row; do
   node_index_lts="${row[9]}"
   [[ "$node_index_version" = "version" ]] && continue # skip tsv header
   [[ "$NODE_LTS_VERSION" = "latest" ]] && break # first line is latest
+  [[ "$NODE_LTS_VERSION" = "$node_index_version" ]] && break # match full version if specified
   [[ "$NODE_LTS_VERSION" = "$node_index_major_version" ]] && break # case insensitive compare
 done < node_index.tab
 

.evergreen/run-benchmarks.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh"
+
+npm run check:bench

.evergreen/run-tests.sh

@@ -2,7 +2,7 @@
 
 source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh"
 
-case $1 in
+case "${TEST_TARGET}" in
   "node")
     npm run check:coverage
     ;;

package.json

@@ -103,6 +103,7 @@
     "check:tsd": "npm run build:dts && tsd",
     "check:web": "WEB=true mocha test/node",
     "check:web-no-bigint": "WEB=true NO_BIGINT=true mocha test/node",
+    "check:bench":"cd test/bench && npx tsc && node ./lib/index.js && mv benchmarks.json ../../.",
     "build:ts": "node ./node_modules/typescript/bin/tsc",
     "build:dts": "npm run build:ts && api-extractor run --typescript-compiler-folder node_modules/typescript --local && node etc/clean_definition_files.cjs",
     "build:bundle": "rollup -c rollup.config.mjs",

test/bench/.gitignore

@@ -0,0 +1,2 @@
+lib
+node_modules

test/bench/src/index.ts

@@ -0,0 +1,32 @@
+import { getStringDeserializationSuite } from './suites/string_deserialization';
+import { type PerfSendData } from './util';
+import { writeFile } from 'fs';
+import { cpus, totalmem } from 'os';
+
+const hw = cpus();
+const platform = { name: hw[0].model, cores: hw.length, ram: `${totalmem() / 1024 ** 3}GiB` };
+
+const results: PerfSendData[][] = [];
+
+console.log(
+  [
+    `\n- cpu: ${platform.name}`,
+    `- cores: ${platform.cores}`,
+    `- os: ${process.platform}`,
+    `- ram: ${platform.ram}`
+  ].join('\n')
+);
+
+for (const suite of [getStringDeserializationSuite()]) {
+  suite.run();
+  results.push(suite.results);
+}
+
+writeFile('benchmarks.json', JSON.stringify(results.flat()), err => {
+  if (err) {
+    console.error(err);
+    process.exit(1);
+  }
+
+  console.log('Wrote results to benchmarks.json');
+});

test/bench/src/suite.ts

@@ -0,0 +1,46 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { Task } from './task';
+import { type PerfSendData } from './util';
+
+export class Suite {
+  name: string;
+  tasks: Task[];
+  results: PerfSendData[];
+  constructor(name: string) {
+    this.name = name;
+    this.tasks = [];
+    this.results = [];
+  }
+
+  task(opts: {
+    name: string;
+    data: any;
+    fn: (data: any) => void;
+    iterations: number;
+    resultUnit?: string;
+    transform?: (x: number) => number;
+    args?: Record<string, string>;
+  }) {
+    this.tasks.push(
+      new Task(
+        this,
+        opts.name,
+        opts.data,
+        opts.fn,
+        opts.iterations,
+        opts.resultUnit,
+        opts.transform,
+        opts.args
+      )
+    );
+    return this;
+  }
+
+  run() {
+    console.log(`Running suite: ${this.name}`);
+
+    for (const task of this.tasks) {
+      task.run();
+    }
+  }
+}

test/bench/src/suites/string_deserialization.ts

@@ -0,0 +1,31 @@
+import { Suite } from '../suite';
+import * as BSON from '../../../../';
+
+export function getStringDeserializationSuite(): Suite {
+  const DOC: Uint8Array = BSON.serialize({
+    nextBatch: Array.from({ length: 1000 }, () => {
+      return {
+        _id: new BSON.ObjectId(),
+        arrayField: Array.from({ length: 20 }, (_, i) => `5e99f3f5d3ab06936d36000${i}`)
+      };
+    })
+  });
+  const suite = new Suite('string deserialization');
+  for (const utf8Validation of [true, false]) {
+    suite.task({
+      name: `stringDeserializationUTF8${utf8Validation ? 'On' : 'Off'}-${
+        process.env.WEB === 'true' ? 'web' : 'node'
+      }`,
+      data: DOC,
+      fn: serializedDoc =>
+        BSON.deserialize(serializedDoc, { validation: { utf8: utf8Validation } }),
+      iterations: 10_000,
+      resultUnit: 'megabytes_per_second',
+      transform: (runtimeMS: number) => {
+        return DOC.byteLength / 1024 ** 2 / (runtimeMS / 1000);
+      }
+    });
+  }
+
+  return suite;
+}

test/bench/src/task.ts

@@ -0,0 +1,66 @@
+import { type Suite } from './suite';
+import { convertToPerfSendFormat } from './util';
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { performance } from 'perf_hooks';
+
+export class Task {
+  name: string;
+  parent: Suite;
+  data: any;
+  fn: (data: any) => void;
+  iterations: number;
+  transform?: (x: number) => number;
+  resultUnit: string;
+  args?: Record<string, string>;
+
+  constructor(
+    parent: Suite,
+    name: string,
+    data: any,
+    fn: (data: any) => void,
+    iterations: number,
+    resultUnit?: string,
+    transform?: (x: number) => number,
+    args?: Record<string, string>
+  ) {
+    this.parent = parent;
+    this.name = name;
+    this.iterations = iterations;
+    this.data = data;
+    this.fn = fn;
+    this.transform = transform;
+    this.args = args;
+    this.resultUnit = resultUnit ? resultUnit : 'ms';
+  }
+
+  // TODO: Ensure that each task runs on a separate node process
+  run() {
+    console.log(`\t ${this.name} - iters: ${this.iterations}`);
+    const data = this.data;
+    const fn = this.fn;
+    // Warmup
+    for (let i = 0; i < this.iterations; i++) {
+      fn(data);
+    }
+    const results: number[] = [];
+
+    for (let i = 0; i < this.iterations; i++) {
+      const start = performance.now();
+      fn(data);
+      const end = performance.now();
+      results.push(end - start); // ms
+    }
+    this.parent.results.push(
+      convertToPerfSendFormat(
+        this.name,
+        [
+          {
+            name: this.resultUnit,
+            results: this.transform ? results.map(this.transform) : results
+          }
+        ],
+        this.args
+      )
+    );
+  }
+}

test/bench/src/util.ts

@@ -0,0 +1,29 @@
+export type PerfSendData = {
+  info: {
+    test_name: string;
+    tags?: string[];
+    args?: Record<string, string>;
+  };
+  metrics: { name: string; value: number }[];
+};
+
+export function convertToPerfSendFormat(
+  benchmarkName: string,
+  metrics: {
+    name: string;
+    results: number[];
+  }[],
+  args?: Record<string, string>
+): PerfSendData {
+  return {
+    info: {
+      test_name: benchmarkName.replaceAll(' ', '_'),
+      tags: ['js-bson'],
+      args: args
+    },
+    metrics: metrics.map(({ name, results }) => ({
+      name,
+      value: results.reduce((acc, x) => acc + x, 0) / results.length
+    }))
+  };
+}

test/bench/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "allowJs": false,
+    "checkJs": false,
+    "strict": true,
+    "alwaysStrict": true,
+    "target": "es2021",
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "skipLibCheck": true,
+    "lib": [
+      "es2021"
+    ],
+    "outDir": "./lib",
+    "importHelpers": false,
+    "noEmitHelpers": false,
+    "noEmitOnError": true,
+    "emitDeclarationOnly": false,
+    "sourceMap": true,
+    "inlineSourceMap": false,
+    "inlineSources": false,
+    "types": ["node"]
+  },
+  "include": [
+    "./src/**/*.ts"
+  ]
+}