Integration test runner (#17)

* Serverless integration test runner

* Allow integration test pipeline to comment on PRs

Author: JoshMock

.buildkite/Dockerfile

@@ -1,7 +1,7 @@
 ARG NODE_VERSION=${NODE_VERSION:-18}
 FROM node:$NODE_VERSION
 
-# Install required tools
+# Install zip util
 RUN apt-get clean -y && \
     apt-get -qy update && \
     apt-get -y install zip && \
@@ -10,5 +10,7 @@ RUN apt-get clean -y && \
 
 WORKDIR /usr/src/app
 
-COPY . .
+COPY package*.json .
 RUN npm install --production=false
+
+COPY . .

.buildkite/create-serverless.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export EC_REGISTER_BACKEND=appex-qa-team-cluster
+export EC_ENV=qa
+export EC_REGION=aws-eu-west-1
+# Using BUILDKITE_JOB_ID for the name to make it unique:
+export EC_PROJECT_NAME="$EC_PROJECT_PREFIX-$BUILDKITE_JOB_ID"
+
+# fetch cloud creds used by qaf
+CLOUD_ACCESS_KEY=$(vault read -field="$EC_ENV" "$CLOUD_CREDENTIALS_PATH")
+echo "{\"api_key\":{\"$EC_ENV\":\"$CLOUD_ACCESS_KEY\"}}" > "$(pwd)/cloud.json"
+
+run_qaf() {
+  cmd=$1
+  docker run --rm \
+         -e EC_REGISTER_BACKEND \
+         -e EC_ENV \
+         -e EC_REGION \
+         -e EC_PROJECT_NAME \
+         -e VAULT_TOKEN \
+         -e BUILDKITE \
+         -v "$(pwd)/cloud.json:/root/.elastic/cloud.json" \
+         docker.elastic.co/employees/dolaru/qaf:latest \
+         bash -c "$cmd"
+}
+
+# ensure serverless instance is deleted even if script errors
+cleanup() {
+  echo -e "--- :elasticsearch: :broom: :sparkles: Tear down serverless instance $EC_PROJECT_NAME"
+  run_qaf 'qaf elastic-cloud projects delete'
+  rm -rf "$(pwd)/cloud.json"
+}
+
+start_serverless() {
+  echo -e "--- :elasticsearch: Start serverless instance $EC_PROJECT_NAME"
+
+  run_qaf "qaf elastic-cloud projects create --project-type elasticsearch"
+  deployment=$(run_qaf "qaf elastic-cloud projects describe $EC_PROJECT_NAME --as-json --show-credentials")
+
+  # Set ELASTICSEARCH_URL and API_KEY variables
+  ES_API_SECRET_KEY=$(echo "$deployment" | jq -r '.credentials.api_key')
+  export ES_API_SECRET_KEY
+  ELASTICSEARCH_URL=$(echo "$deployment" | jq -r '.elasticsearch.url')
+  export ELASTICSEARCH_URL
+
+  echo -e "--- :computer: Environment variables"
+  echo -e "ELASTICSEARCH_URL $ELASTICSEARCH_URL"
+}

.buildkite/functions/cleanup.sh

@@ -1,3 +1,32 @@
+---
 steps:
-  - label: ":elasticsearch: :javascript: Elasticsearch Serverless Node.js"
-    command: "echo 'Hello, world!'"
+  - key: integration
+    label: ":elasticsearch: :javascript: Elasticsearch Serverless Node.js integration tests"
+    agents:
+      provider: "gcp"
+    matrix:
+      setup:
+        nodejs:
+          - "16"
+          - "18"
+          - "20"
+    env:
+      NODE_VERSION: "{{ matrix.nodejs }}"
+      EC_PROJECT_PREFIX: "elasticsearch-serverless-js-node{{ matrix.nodejs }}"
+      CLOUD_CREDENTIALS_PATH: "secret/ci/elastic-elasticsearch-serverless-js/cloud-access"
+      GITHUB_TOKEN_PATH: "secret/ci/elastic-elasticsearch-serverless-js/github-token"
+    command: "./.buildkite/run-tests.sh"
+    artifact_paths: "./junit-output/junit-*.xml"
+  - wait: ~
+    continue_on_failure: true
+  - key: junit
+    label: ":junit: Test results"
+    agents:
+      provider: "gcp"
+      image: family/core-ubuntu-2204
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "junit-output/junit-*.xml"
+          job-uuid-file-pattern: "junit-(.*).xml"
+          fail-build-on-error: true
+          failure-format: file

.buildkite/functions/imports.sh

@@ -6,26 +6,34 @@ script_path=$(dirname "$(realpath -s "$0")")
 set -euo pipefail
 repo=$(pwd)
 
-export NODE_VERSION=${NODE_VERSION:-18}
+export NODE_VERSION=${NODE_VERSION:-16}
 
-echo "--- :javascript: Building Docker image"
+echo "--- :docker: Building Docker image"
 docker build \
        --file "$script_path/Dockerfile" \
-       --tag elastic/elasticsearch-js \
+       --tag elastic/elasticsearch-serverless-js \
        --build-arg NODE_VERSION="$NODE_VERSION" \
        .
 
-echo "--- :javascript: Running $TEST_SUITE tests"
+echo "--- :javascript: Running tests"
+
+GITHUB_TOKEN=$(vault read -field=token "$GITHUB_TOKEN_PATH")
+export GITHUB_TOKEN
+
 mkdir -p "$repo/junit-output"
 docker run \
-       --network="${network_name}" \
-       --env "TEST_ES_SERVER=${elasticsearch_url}" \
-       --env "ELASTIC_PASSWORD=${elastic_password}" \
-       --env "TEST_SUITE=${TEST_SUITE}" \
-       --env "ELASTIC_USER=elastic" \
-       --env "BUILDKITE=true" \
-       --volume "$repo/junit-output:/junit-output" \
-       --name elasticsearch-js \
+       -e "ELASTICSEARCH_URL" \
+       -e "ES_API_SECRET_KEY" \
+       -e "GITHUB_TOKEN" \
+       -e "BUILDKITE" \
+       --volume "$repo/junit-output:/usr/src/app/junit-output" \
+       --name elasticsearch-serverless-js \
        --rm \
-       elastic/elasticsearch-js \
-       bash -c "npm run test:integration; [ -f ./$TEST_SUITE-report-junit.xml ] && mv ./$TEST_SUITE-report-junit.xml /junit-output/junit-$BUILDKITE_JOB_ID.xml || echo 'No JUnit artifact found'"
+       elastic/elasticsearch-serverless-js \
+       bash -c "npm run test:integration"
+
+if [ -f "$repo/junit-output/serverless-report-junit.xml" ]; then
+  mv "$repo/junit-output/serverless-report-junit.xml" "$repo/junit-output/junit-$BUILDKITE_JOB_ID.xml"
+else
+  echo 'No JUnit artifact found'
+fi