ci: Expand integration test runner logic (#24)

* Improve compatibility with REST YAML test format

* Dynamic label for integration test steps

* Simplify integration test runner hierarchy

We were organizing into APIs and files, but we can just run a flat array
of test files.

* Log failure count for convenience

* Use commonjs version of globby

* Update naming

* Reinsert stray line that got deleted

Author: JoshMock

.buildkite/pipeline.yml

@@ -1,7 +1,7 @@
 ---
 steps:
   - key: integration
-    label: ":elasticsearch: :javascript: Elasticsearch Serverless Node.js integration tests"
+    label: ":elasticsearch: :javascript: Elasticsearch Serverless Node.js v{{ matrix.nodejs }} integration tests"
     agents:
       provider: "gcp"
     matrix:

.ci/make.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # ------------------------------------------------------- #
 #
-# Build entry script for elasticsearch-js
+# Build entry script for elasticsearch-serverless-js
 #
 # Must be called: ./.ci/make.sh <target> <params>
 #
@@ -33,7 +33,7 @@ VERSION=$2
 STACK_VERSION=$VERSION
 set -euo pipefail
 
-product="elastic/elasticsearch-js"
+product="elastic/elasticsearch-serverless-js"
 output_folder=".ci/output"
 codegen_folder=".ci/output"
 OUTPUT_DIR="$repo/${output_folder}"
@@ -148,7 +148,7 @@ if [[ -z "${BUILDKITE+x}" ]] || [[ -z "${CI+x}" ]]; then
     --volume "$repo:/usr/src/app" \
     --volume "$(realpath $repo/../elastic-client-generator-js):/usr/src/elastic-client-generator-js" \
     --env "WORKFLOW=$WORKFLOW" \
-    --name make-elasticsearch-js \
+    --name make-elasticsearch-serverless-js \
     --rm \
     $product \
     /bin/bash -c "mkdir -p /usr/src/elastic-client-generator-js/output && \
@@ -159,7 +159,7 @@ else
     --volume "$repo:/usr/src/app" \
     --volume /usr/src/app/node_modules \
     --env "WORKFLOW=$WORKFLOW" \
-    --name make-elasticsearch-js \
+    --name make-elasticsearch-serverless-js \
     --rm \
     $product \
     /bin/bash -c "cd /usr/src && \

package.json

@@ -59,6 +59,7 @@
     "chai": "^4.3.10",
     "cross-zip": "^4.0.0",
     "desm": "^1.3.0",
+    "globby": "^11.1.0",
     "js-yaml": "^4.1.0",
     "license-checker": "^25.0.1",
     "node-fetch": "^2.7.0",

test/integration/index.js

@@ -29,6 +29,7 @@ const { join, sep } = require('path')
 const yaml = require('js-yaml')
 const minimist = require('minimist')
 const ms = require('ms')
+const globby = require('globby')
 const { Client } = require('../../index')
 const build = require('./test-runner')
 const createJunitReporter = require('./reporter')
@@ -45,18 +46,13 @@ const options = minimist(process.argv.slice(2), {
   string: ['suite', 'test'],
 })
 
-const getAllFiles = dir => {
-  return readdirSync(dir)
-    .reduce((files, file) => {
-      const name = join(dir, file)
-      if (statSync(name).isDirectory()) {
-        return [...files, ...getAllFiles(name)]
-      } else if (!name.endsWith('.yaml') && !name.endsWith('.yml')) {
-        return files
-      } else {
-        return [...files, name]
-      }
-    }, [])
+const getAllFiles = async dir => {
+  const files = await globby(dir, {
+    expandDirectories: {
+      extensions: ['yml', 'yaml']
+    }
+  })
+  return files.sort()
 }
 
 function runner (opts = {}) {
@@ -89,112 +85,88 @@ async function start ({ client }) {
     pass: 0,
     assertions: 0
   }
-  const folders = getAllFiles(yamlFolder)
-    .reduce((arr, file) => {
-      const path = file.slice(file.indexOf('/tests'), file.lastIndexOf('/'))
-      let inserted = false
-      for (let i = 0; i < arr.length; i++) {
-        if (arr[i][0].includes(path)) {
-          inserted = true
-          arr[i].push(file)
-          break
-        }
-      }
-      if (!inserted) arr.push([file])
-      return arr
-    }, [])
+  const files = await getAllFiles(yamlFolder)
 
   const totalTime = now()
-  for (const folder of folders) {
+  for (const file of files) {
     // pretty name
-    const apiName = folder[0].split(`${sep}tests${sep}`)[1]
+    const apiName = file.split(`${sep}tests${sep}`)[1]
 
     log('Testing ' + apiName)
-    const apiTime = now()
-
-    for (const file of folder) {
-      const testRunner = build({ client })
-      const fileTime = now()
-      const data = readFileSync(file, 'utf8')
-
-      // get the test yaml (as object), some file has multiple yaml documents inside,
-      // every document is separated by '---', so we split on the separator
-      // and then we remove the empty strings, finally we parse them
-      const tests = data
-        .split('\n---\n')
-        .map(s => s.trim())
-        // empty strings
-        .filter(Boolean)
-        .map(parse)
-        // null values
-        .filter(Boolean)
-
-      // get setup and teardown if present
-      let setupTest = null
-      let teardownTest = null
-      for (const test of tests) {
-        if (test.setup) setupTest = test.setup
-        if (test.teardown) teardownTest = test.teardown
-      }
+    const testRunner = build({ client })
+    const fileTime = now()
+    const data = readFileSync(file, 'utf8')
+
+    // get the test yaml (as object), some file has multiple yaml documents inside,
+    // every document is separated by '---', so we split on the separator
+    // and then we remove the empty strings, finally we parse them
+    const tests = data
+      .split('\n---\n')
+      .map(s => s.trim())
+      // empty strings
+      .filter(Boolean)
+      .map(parse)
+      // null values
+      .filter(Boolean)
+
+    // get setup and teardown if present
+    let setupTest = null
+    let teardownTest = null
+    for (const test of tests) {
+      if (test.setup) setupTest = test.setup
+      if (test.teardown) teardownTest = test.teardown
+    }
 
-      const cleanPath = file.slice(file.lastIndexOf(apiName))
-
-      // skip if --suite CLI arg doesn't match
-      if (options.suite && !cleanPath.endsWith(options.suite)) continue
-
-      log('    ' + cleanPath)
-      const junitTestSuite = junitTestSuites.testsuite(apiName.slice(1) + ' - ' + cleanPath)
-
-      for (const test of tests) {
-        const testTime = now()
-        const name = Object.keys(test)[0]
-
-        // skip setups, teardowns and anything that doesn't match --test flag when present
-        if (name === 'setup' || name === 'teardown') continue
-        if (options.test && !name.endsWith(options.test)) continue
-
-        const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}/${cleanPath}`)
-
-        stats.total += 1
-        log('        - ' + name)
-        try {
-          await testRunner.run(setupTest, test[name], teardownTest, stats, junitTestCase)
-          stats.pass += 1
-        } catch (err) {
-          junitTestCase.failure(err)
-          junitTestCase.end()
-          junitTestSuite.end()
-          junitTestSuites.end()
-          generateJunitXmlReport(junit, 'serverless')
-          console.error(err)
-
-          if (options.bail) {
-            process.exit(1)
-          } else {
-            continue
-          }
-        }
-        const totalTestTime = now() - testTime
+    const cleanPath = file.slice(file.lastIndexOf(apiName))
+
+    // skip if --suite CLI arg doesn't match
+    if (options.suite && !cleanPath.endsWith(options.suite)) continue
+
+    const junitTestSuite = junitTestSuites.testsuite(apiName.slice(1) + ' - ' + cleanPath)
+
+    for (const test of tests) {
+      const testTime = now()
+      const name = Object.keys(test)[0]
+
+      // skip setups, teardowns and anything that doesn't match --test flag when present
+      if (name === 'setup' || name === 'teardown') continue
+      if (options.test && !name.endsWith(options.test)) continue
+
+      const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}/${cleanPath}`)
+
+      stats.total += 1
+      log('  - ' + name)
+      try {
+        await testRunner.run(setupTest, test[name], teardownTest, stats, junitTestCase)
+        stats.pass += 1
+      } catch (err) {
+        junitTestCase.failure(err)
         junitTestCase.end()
-        if (totalTestTime > MAX_TEST_TIME) {
-          log('          took too long: ' + ms(totalTestTime))
+        junitTestSuite.end()
+        junitTestSuites.end()
+        generateJunitXmlReport(junit, 'serverless')
+        console.error(err)
+
+        if (options.bail) {
+          process.exit(1)
         } else {
-          log('          took: ' + ms(totalTestTime))
+          continue
         }
       }
-      junitTestSuite.end()
-      const totalFileTime = now() - fileTime
-      if (totalFileTime > MAX_FILE_TIME) {
-        log(`    ${cleanPath} took too long: ` + ms(totalFileTime))
+      const totalTestTime = now() - testTime
+      junitTestCase.end()
+      if (totalTestTime > MAX_TEST_TIME) {
+        log('    took too long: ' + ms(totalTestTime))
       } else {
-        log(`    ${cleanPath} took: ` + ms(totalFileTime))
+        log('    took: ' + ms(totalTestTime))
       }
     }
-    const totalApiTime = now() - apiTime
-    if (totalApiTime > MAX_API_TIME) {
-      log(`${apiName} took too long: ` + ms(totalApiTime))
+    junitTestSuite.end()
+    const totalFileTime = now() - fileTime
+    if (totalFileTime > MAX_FILE_TIME) {
+      log(`  ${cleanPath} took too long: ` + ms(totalFileTime))
     } else {
-      log(`${apiName} took: ` + ms(totalApiTime))
+      log(`  ${cleanPath} took: ` + ms(totalFileTime))
     }
   }
   junitTestSuites.end()
@@ -204,6 +176,7 @@ async function start ({ client }) {
   - Total: ${stats.total}
   - Skip: ${stats.skip}
   - Pass: ${stats.pass}
+  - Fail: ${stats.total - stats.pass}
   - Assertions: ${stats.assertions}
   `)
 }

test/integration/test-runner.js

@@ -420,10 +420,10 @@ function build (opts = {}) {
         const key = Object.keys(action.match)[0]
         match(
           // in some cases, the yaml refers to the body with an empty string
-          key === '$body' || key === ''
+          key.split('.')[0] === '$body' || key === ''
             ? response
             : delve(response, fillStashedValues(key)),
-          key === '$body'
+          key.split('.')[0] === '$body'
             ? action.match[key]
             : fillStashedValues(action.match)[key],
           action.match
@@ -544,6 +544,8 @@ function match (val1, val2, action) {
     // 'm' adds the support for multiline regex
     assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}`)
   // everything else
+  } else if (typeof val1 === 'string' && typeof val2 === 'string') {
+    assert.include(val1, val2, `should match pattern provided: ${val2}, but got: ${val1}`)
   } else {
     assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`)
   }
@@ -640,6 +642,10 @@ function length (val, len) {
  */
 function parseDo (action) {
   action = JSON.parse(JSON.stringify(action))
+
+  if (typeof action === 'string') action = {[action]: {}}
+  if (Array.isArray(action)) action = action[0]
+
   return Object.keys(action).reduce((acc, val) => {
     switch (val) {
       case 'catch':