[Backport 8.7] Fixes to YAML REST integration test suite runner (#1840)

Co-authored-by: Josh Mock <[email protected]>

Author: github-actions[bot]

.ci/run-elasticsearch.sh

@@ -22,12 +22,12 @@
 # - Use https only when TEST_SUITE is "platinum", when "free" use http
 # - Set xpack.security.enabled=false for "free" and xpack.security.enabled=true for "platinum"
 
-script_path=$(dirname $(realpath -s $0))
-source $script_path/functions/imports.sh
+script_path=$(dirname "$(realpath -s "$0")")
+source "$script_path/functions/imports.sh"
 set -euo pipefail
 
-echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
-cleanup_node $es_node_name
+echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on separate terminals \033[0m"
+cleanup_node "$es_node_name"
 
 master_node_name=${es_node_name}
 cluster_name=${moniker}${suffix}

Makefile

@@ -0,0 +1,12 @@
+.PHONY: integration-setup
+integration-setup: integration-cleanup
+	DETACH=true .ci/run-elasticsearch.sh
+
+.PHONY: integration-cleanup
+integration-cleanup:
+	docker stop instance || true
+	docker volume rm instance-rest-test-data || true
+
+.PHONY: integration
+integration: integration-setup
+	npm run test:integration

scripts/utils/generateApis.js

@@ -228,7 +228,7 @@ function generateSingleApi (version, spec, common) {
 
     ${genUrlValidation(paths, api)}
 
-    let { ${genQueryBlacklist(false)}, ...querystring } = params
+    let { ${genQueryDenylist(false)}, ...querystring } = params
     querystring = snakeCaseKeys(acceptedQuerystring, snakeCase, querystring)
 
     let path = ''
@@ -316,20 +316,20 @@ function generateSingleApi (version, spec, common) {
     }, {})
   }
 
-  function genQueryBlacklist (addQuotes = true) {
+  function genQueryDenylist (addQuotes = true) {
     const toCamelCase = str => {
       return str[0] === '_'
         ? '_' + str.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
         : str.replace(/_([a-z])/g, k => k[1].toUpperCase())
     }
 
-    const blacklist = ['method', 'body']
+    const denylist = ['method', 'body']
     parts.forEach(p => {
       const camelStr = toCamelCase(p)
-      if (camelStr !== p) blacklist.push(`${camelStr}`)
-      blacklist.push(`${p}`)
+      if (camelStr !== p) denylist.push(`${camelStr}`)
+      denylist.push(`${p}`)
     })
-    return addQuotes ? blacklist.map(q => `'${q}'`) : blacklist
+    return addQuotes ? denylist.map(q => `'${q}'`) : denylist
   }
 
   function buildPath () {

test/integration/README.md

@@ -5,8 +5,8 @@
 Yes.
 
 ## Background
-Elasticsearch offers its entire API via HTTP REST endpoints. You can find the whole API specification for every version [here](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/api).<br/>
-To support different languages at the same time, the Elasticsearch team decided to provide a [YAML specification](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/test) to test every endpoint, body, headers, warning, error and so on.<br/>
+Elasticsearch offers its entire API via HTTP REST endpoints. You can find the whole API specification for every version [here](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/api).<br/>
+To support different languages at the same time, the Elasticsearch team decided to provide a [YAML specification](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test) to test every endpoint, body, headers, warning, error and so on.<br/>
 This testing suite uses that specification to generate the test for the specified version of Elasticsearch on the fly.
 
 ## Run
@@ -20,20 +20,45 @@ Once the Elasticsearch repository has been cloned, the testing suite will connec
 
 The specification does not allow the test to be run in parallel, so it might take a while to run the entire testing suite; on my machine, `MacBookPro15,2 core i7 2.7GHz 16GB of RAM` it takes around four minutes.
 
+### Running locally
+
+If you want to run the integration tests on your development machine, you must have an Elasticsearch instance running first.
+A local instance can be spun up in a Docker container by running the [`.ci/run-elasticsearch.sh`](/.ci/run-elasticsearch.sh) script.
+This is the same script CI jobs use to run Elasticsearch for integration tests, so your results should be relatively consistent.
+
+To simplify the process of starting a container, testing, and cleaning up the container, you can run the `make integration` target:
+
+```sh
+# set some parameters
+export STACK_VERSION=8.7.0
+export TEST_SUITE=free # can be `free` or `platinum`
+make integration
+```
+
+If Elasticsearch doesn't come up, run `make integration-cleanup` and then `DETACH=false .ci/run-elasticsearch.sh` manually to read the startup logs.
+
+If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144 | sudo tee -a /etc/sysctl.conf` to update the setting permanently.
+
 ### Exit on the first failure
-Bu default the suite will run all the test, even if one assertion has failed. If you want to stop the test at the first failure, use the bailout option:
+
+By default the suite will run all the tests, even if one assertion has failed. If you want to stop the test at the first failure, use the bailout option:
+
 ```sh
 npm run test:integration -- --bail
 ```
 
 ### Calculate the code coverage
+
 If you want to calculate the code coverage just run the testing suite with the following parameters, once the test ends, it will open a browser window with the results.
+
 ```sh
 npm run test:integration -- --cov --coverage-report=html
 ```
 
 ## How does this thing work?
+
 At first sight, it might seem complicated, but once you understand what the moving parts are, it's quite easy.
+
 1. Connects to the given Elasticsearch instance
 1. Gets the ES version and build hash
 1. Checkout to the given hash (and clone the repository if it is not present)
@@ -46,7 +71,4 @@ At first sight, it might seem complicated, but once you understand what the movi
 
 Inside the `index.js` file, you will find the connection, cloning, reading and parsing part of the test, while inside the `test-runner.js` file you will find the function to handle the assertions. Inside `test-runner.js`, we use a [queue](https://github.com/delvedor/workq) to be sure that everything is run in the correct order.
 
-Checkout the [rest-api-spec readme](https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc) if you want to know more about how the assertions work.
-
-#### Why are we running the test with the `--harmony` flag?
-Because on Node v6 the regex lookbehinds are not supported.
+Check out the [rest-api-spec readme](https://github.com/elastic/elasticsearch/blob/main/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc) if you want to know more about how the assertions work.

test/integration/index.js

@@ -27,6 +27,7 @@ process.on('unhandledRejection', function (err) {
 const { writeFileSync, readFileSync, readdirSync, statSync } = require('fs')
 const { join, sep } = require('path')
 const yaml = require('js-yaml')
+const minimist = require('minimist')
 const ms = require('ms')
 const { Client } = require('../../index')
 const { kProductCheck } = require('@elastic/transport/lib/symbols')
@@ -42,12 +43,24 @@ const MAX_API_TIME = 1000 * 90
 const MAX_FILE_TIME = 1000 * 30
 const MAX_TEST_TIME = 1000 * 3
 
+const options = minimist(process.argv.slice(2), {
+  boolean: ['bail']
+})
+
 const freeSkips = {
   // not supported yet
   '/free/cluster.desired_nodes/10_basic.yml': ['*'],
+
+  // Cannot find methods on `Internal` object
+  '/free/cluster.desired_balance/10_basic.yml': ['*'],
+  '/free/cluster.desired_nodes/20_dry_run.yml': ['*'],
+  '/free/cluster.prevalidate_node_removal/10_basic.yml': ['*'],
+
   '/free/health/30_feature.yml': ['*'],
   '/free/health/40_useractions.yml': ['*'],
-  // the v8 client never sends the scroll_id in querystgring,
+  '/free/health/40_diagnosis.yml': ['Diagnosis'],
+
+  // the v8 client never sends the scroll_id in querystring,
   // the way the test is structured causes a security exception
   'free/scroll/10_basic.yml': ['Body params override query string'],
   'free/scroll/11_clear.yml': [
@@ -56,80 +69,99 @@ const freeSkips = {
   ],
   'free/cat.allocation/10_basic.yml': ['*'],
   'free/cat.snapshots/10_basic.yml': ['Test cat snapshots output'],
+
   // TODO: remove this once 'arbitrary_key' is implemented
   // https://github.com/elastic/elasticsearch/pull/41492
   'indices.split/30_copy_settings.yml': ['*'],
   'indices.stats/50_disk_usage.yml': ['Disk usage stats'],
   'indices.stats/60_field_usage.yml': ['Field usage stats'],
+
   // skipping because we are booting ES with `discovery.type=single-node`
   // and this test will fail because of this configuration
   'nodes.stats/30_discovery.yml': ['*'],
+
   // the expected error is returning a 503,
   // which triggers a retry and the node to be marked as dead
   'search.aggregation/240_max_buckets.yml': ['*'],
+
   // long values and json do not play nicely together
   'search.aggregation/40_range.yml': ['Min and max long range bounds'],
+
   // the yaml runner assumes that null means "does not exists",
   // while null is a valid json value, so the check will fail
   'search/320_disallow_queries.yml': ['Test disallow expensive queries'],
-  'free/tsdb/90_unsupported_operations.yml': ['noop update']
+  'free/tsdb/90_unsupported_operations.yml': ['noop update'],
 }
-const platinumBlackList = {
+
+const platinumDenyList = {
   'api_key/10_basic.yml': ['Test get api key'],
   'api_key/20_query.yml': ['*'],
   'api_key/11_invalidation.yml': ['Test invalidate api key by realm name'],
   'analytics/histogram.yml': ['Histogram requires values in increasing order'],
+
   // this two test cases are broken, we should
   // return on those in the future.
   'analytics/top_metrics.yml': [
     'sort by keyword field fails',
     'sort by string script fails'
   ],
+
   'cat.aliases/10_basic.yml': ['Empty cluster'],
   'index/10_with_id.yml': ['Index with ID'],
   'indices.get_alias/10_basic.yml': ['Get alias against closed indices'],
   'indices.get_alias/20_empty.yml': ['Check empty aliases when getting all aliases via /_alias'],
   'text_structure/find_structure.yml': ['*'],
+
   // https://github.com/elastic/elasticsearch/pull/39400
   'ml/jobs_crud.yml': ['Test put job with id that is already taken'],
+
   // object keys must me strings, and `0.0.toString()` is `0`
   'ml/evaluate_data_frame.yml': [
     'Test binary_soft_classifition precision',
     'Test binary_soft_classifition recall',
     'Test binary_soft_classifition confusion_matrix'
   ],
+
   // it gets random failures on CI, must investigate
   'ml/set_upgrade_mode.yml': [
     'Attempt to open job when upgrade_mode is enabled',
     'Setting upgrade mode to disabled from enabled'
   ],
+
   // The cleanup fails with a index not found when retrieving the jobs
   'ml/get_datafeed_stats.yml': ['Test get datafeed stats when total_search_time_ms mapping is missing'],
   'ml/bucket_correlation_agg.yml': ['Test correlation bucket agg simple'],
+
   // start should be a string
   'ml/jobs_get_result_overall_buckets.yml': ['Test overall buckets given epoch start and end params'],
+
   // this can't happen with the client
   'ml/start_data_frame_analytics.yml': ['Test start with inconsistent body/param ids'],
   'ml/stop_data_frame_analytics.yml': ['Test stop with inconsistent body/param ids'],
   'ml/preview_datafeed.yml': ['*'],
+
   // Investigate why is failing
   'ml/inference_crud.yml': ['*'],
   'ml/categorization_agg.yml': ['Test categorization aggregation with poor settings'],
   'ml/filter_crud.yml': ['*'],
+
   // investigate why this is failing
   'monitoring/bulk/10_basic.yml': ['*'],
   'monitoring/bulk/20_privileges.yml': ['*'],
   'license/20_put_license.yml': ['*'],
   'snapshot/10_basic.yml': ['*'],
   'snapshot/20_operator_privileges_disabled.yml': ['*'],
+
   // the body is correct, but the regex is failing
   'sql/sql.yml': ['Getting textual representation'],
   'searchable_snapshots/10_usage.yml': ['*'],
   'service_accounts/10_basic.yml': ['*'],
+
   // we are setting two certificates in the docker config
   'ssl/10_basic.yml': ['*'],
   'token/10_basic.yml': ['*'],
   'token/11_invalidation.yml': ['*'],
+
   // very likely, the index template has not been loaded yet.
   // we should run a indices.existsTemplate, but the name of the
   // template may vary during time.
@@ -147,16 +179,20 @@ const platinumBlackList = {
   'transforms_stats.yml': ['*'],
   'transforms_stats_continuous.yml': ['*'],
   'transforms_update.yml': ['*'],
+
   // js does not support ulongs
   'unsigned_long/10_basic.yml': ['*'],
   'unsigned_long/20_null_value.yml': ['*'],
   'unsigned_long/30_multi_fields.yml': ['*'],
   'unsigned_long/40_different_numeric.yml': ['*'],
   'unsigned_long/50_script_values.yml': ['*'],
+
   // the v8 client flattens the body into the parent object
   'platinum/users/10_basic.yml': ['Test put user with different username in body'],
+
   // docker issue?
   'watcher/execute_watch/60_http_input.yml': ['*'],
+
   // the checks are correct, but for some reason the test is failing on js side
   // I bet is because the backslashes in the rg
   'watcher/execute_watch/70_invalid.yml': ['*'],
@@ -170,8 +206,16 @@ const platinumBlackList = {
   'platinum/ml/delete_job_force.yml': ['Test force delete an open job that is referred by a started datafeed'],
   'platinum/ml/evaluate_data_frame.yml': ['*'],
   'platinum/ml/get_datafeed_stats.yml': ['*'],
+
   // start should be a string in the yaml test
-  'platinum/ml/start_stop_datafeed.yml': ['*']
+  'platinum/ml/start_stop_datafeed.yml': ['*'],
+
+  // health API not yet supported
+  '/platinum/health/10_usage.yml': ['*'],
+
+  // ML update_trained_model_deployment not supported yet
+  '/platinum/ml/3rd_party_deployment.yml': ['Test update deployment'],
+  '/platinum/ml/update_trained_model_deployment.yml': ['Test with unknown model id']
 }
 
 function runner (opts = {}) {
@@ -316,7 +360,12 @@ async function start ({ client, isXPack }) {
           junitTestSuites.end()
           generateJunitXmlReport(junit, isXPack ? 'platinum' : 'free')
           console.error(err)
-          process.exit(1)
+
+          if (options.bail) {
+            process.exit(1)
+          } else {
+            continue
+          }
         }
         const totalTestTime = now() - testTime
         junitTestCase.end()
@@ -380,7 +429,8 @@ function generateJunitXmlReport (junit, suite) {
 }
 
 if (require.main === module) {
-  const node = process.env.TEST_ES_SERVER || 'http://elastic:changeme@localhost:9200'
+  const scheme = process.env.TEST_SUITE === 'platinum' ? 'https' : 'http'
+  const node = process.env.TEST_ES_SERVER || `${scheme}://elastic:changeme@localhost:9200`
   const opts = {
     node,
     isXPack: process.env.TEST_SUITE !== 'free'
@@ -395,20 +445,20 @@ const shouldSkip = (isXPack, file, name) => {
     for (let j = 0; j < freeTest.length; j++) {
       if (file.endsWith(list[i]) && (name === freeTest[j] || freeTest[j] === '*')) {
         const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
-        log(`Skipping test ${testName} because is blacklisted in the free test`)
+        log(`Skipping test ${testName} because it is denylisted in the free test suite`)
         return true
       }
     }
   }
 
   if (file.includes('x-pack') || isXPack) {
-    list = Object.keys(platinumBlackList)
+    list = Object.keys(platinumDenyList)
     for (let i = 0; i < list.length; i++) {
-      const platTest = platinumBlackList[list[i]]
+      const platTest = platinumDenyList[list[i]]
       for (let j = 0; j < platTest.length; j++) {
         if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) {
           const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
-          log(`Skipping test ${testName} because is blacklisted in the platinum test`)
+          log(`Skipping test ${testName} because it is denylisted in the platinum test suite`)
           return true
         }
       }