[do not merge] validating buildkite builds in docs repo

.buildkite/build_pr_pipeline.yml

@@ -1,7 +1,33 @@
 steps:
-  - label: ":white_check_mark: Build docs PR"
-    command: |
-      bash .buildkite/scripts/build_pr.sh
+  - key: "build-pr-setup"
+    label: "setup"
+    command: ".buildkite/scripts/build_pr_commit_status.sh pending"
+    plugins:
+      - elastic/vault-secrets#v0.0.2:
+          path: "secret/ci/elastic-docs/docker.elastic.co" #"secret/ci/elastic-docs/docs_preview_cleaner"
+          field: "username" #"value"
+          env_var: "DOCKER_REGISTRY_USERNAME"
+
+      - elastic/vault-secrets#v0.0.2:
+          path: "secret/ci/elastic-docs/docs_preview_cleaner"
+          field: "value"
+          depth: 1
+  - key: "build-pr"
+    label: ":hammer: Build docs PR"
+    command: echo "hello" #".buildkite/scripts/build_pr.sh"
+    depends_on:
+      - step: build-pr-setup
+        allow_failure: true
     agents:
       provider: "gcp"
       image: family/docs-ubuntu-2204
+  - key: "teardown"
+    label: "teardown"
+    command: |
+      if [ $(buildkite-agent step get "outcome" --step "build-pr") == "passed" ]; then
+        .buildkite/scripts/build_pr_commit_status.sh success
+      else
+        .buildkite/scripts/build_pr_commit_status.sh failure
+      fi
+    depends_on:
+      - step: "build-pr"

.buildkite/hooks/pre-command

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euo pipefail
+
+function retry {
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
+
+# Secrets must be redacted
+# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "docs-build-pr" ]];then
+    export GITHUB_TOKEN=$(retry 5 vault kv get -field=value secret/ci/elastic-docs/docs_preview_cleaner)
+fi

.buildkite/scripts/build_pr.sh

@@ -1,2 +1,91 @@
 #!/bin/bash
-echo "nothing to see yet"
+set -euo pipefail
+set +x
+
+# This script should only be invoked by the Buildkite PR bot
+if [ -z ${GITHUB_PR_BRANCH+set} ] || [ -z ${GITHUB_PR_TARGET_BRANCH+set} ] || [ -z ${GITHUB_PR_NUMBER+set} ] || [ -z ${GITHUB_PR_BASE_REPO+set} ];then
+  echo "One of the following env. variable GITHUB_PR_BRANCH, GITHUB_PR_TARGET_BRANCH, GITHUB_PR_NUMBER, GITHUB_PR_BASE_REPO is missing - exiting."
+  exit 1
+fi
+
+# Set some metadata for build filtering capabilities
+# https://buildkite.com/elastic/docs-build-pr/builds?meta_data[repo_pr]=tech-content_123
+buildkite-agent meta-data set "repo_pr" "${GITHUB_PR_BASE_REPO}_${GITHUB_PR_NUMBER}"
+
+rebuild_opt=""
+build_args=""
+TARGET_BRANCH=""
+
+# Define build docs arguments
+if [[ ${GITHUB_PR_COMMENT_VAR_REBUILD_OPT:="unset"} == "rebuild" ]];then
+  rebuild_opt=" --rebuild"
+elif [[ ${GITHUB_PR_COMMENT_VAR_SKIP_OPT:="unset"} == "skiplinkcheck" ]];then
+  build_args+=" --skiplinkcheck"
+  if [[ ${GITHUB_PR_COMMENT_VAR_WARN_OPT:="unset"} == "warnlinkcheck" ]];then
+    build_args+=" --warnlinkcheck"
+  fi
+fi
+
+buildkite-agent \
+    annotate \
+    --style "info" \
+    --context 'docs-info' \
+    "Triggered by a doc change in elastic/$GITHUB_PR_BASE_REPO PR: [#$GITHUB_PR_NUMBER](https://github.com/elastic/$GITHUB_PR_BASE_REPO/pull/$GITHUB_PR_NUMBER)"
+
+
+if [[ "${GITHUB_PR_BASE_REPO}" != 'docs' ]]; then
+  # Buildkite PR bot for repositories other than the `elastic/docs` repo are configured to
+  # always checkout the master branch of the `elastic/docs` repo (where the build logic resides).
+  # We first need to checkout the product repo / branch in a sub directory, that we'll reference
+  # in the build process.
+  echo "Cloning the ${GITHUB_PR_BASE_REPO} PR locally"
+
+  git clone --reference /opt/git-mirrors/elastic-$GITHUB_PR_BASE_REPO \
+    [email protected]:elastic/$GITHUB_PR_BASE_REPO.git ./product-repo
+
+  cd ./product-repo &&
+      git fetch origin pull/$GITHUB_PR_NUMBER/head:$GITHUB_PR_BRANCH &&
+      git switch $GITHUB_PR_BRANCH &&
+      cd ..
+
+  build_args+=" --sub_dir $GITHUB_PR_BASE_REPO:$GITHUB_PR_TARGET_BRANCH:./product-repo"
+else
+  # Buildkite PR bot for the `elastic/docs` repo is configured to checkout the PR directly into the workspace
+  # We don't have to do anything else in this case.
+
+  # Per https://github.com/elastic/docs/issues/1821, always rebuild all
+   # books for PRs to the docs repo, for now.
+   # When https://github.com/elastic/docs/issues/1823 is fixed, this
+   # should be removed and the original behavior restored.
+   rebuild_opt=" --rebuild"
+fi
+
+
+# Set the target branch and preview options
+TARGET_BRANCH="${GITHUB_PR_BASE_REPO}_bk_${GITHUB_PR_NUMBER}"
+PREVIEW_URL="https://${TARGET_BRANCH}.docs-preview.app.elstc.co"
+
+build_cmd="./build_docs --all --keep_hash \
+    --target_repo [email protected]:elastic/built-docs \
+    --reference /opt/git-mirrors/ \
+    --target_branch ${TARGET_BRANCH} \
+    --push \
+    --announce_preview ${PREVIEW_URL}/diff \
+    ${rebuild_opt} \
+    ${build_args}"
+
+echo "The following build command will be used"
+echo $build_cmd
+
+# Kick off the build
+#ssh-agent bash -c "ssh-add && $build_cmd"
+
+buildkite-agent annotate \
+    --style "success" \
+    --context 'docs-info' \
+     --append \
+    "<br>Preview url: ${PREVIEW_URL}"
+
+buildkite-agent meta-data set pr_comment:doc-preview:head " * Documentation preview
+   - 📚 [HTML diff](${PREVIEW_URL}/diff)
+   - 📙 [Preview](${PREVIEW_URL})"

.buildkite/scripts/build_pr_commit_status.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+printenv
+
+# This hook should only be invoked for builds triggered by the Buildkite PR bot
+if [ -z ${GITHUB_PR_OWNER+set} ] || [ -z ${GITHUB_PR_REPO+set} ] || [ -z ${GITHUB_PR_TRIGGERED_SHA+set} ];then
+  exit 0
+fi
+
+status_state=$1
+description=''
+
+case $status_state in
+    pending)
+      description='Build started';;
+    success|failure|error)
+      description='Build finished';;
+    *)
+      echo "Invalid state $status_state"
+      exit 1;;
+esac
+
+githubPublishStatus="https://api.github.com/repos/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}/statuses/${GITHUB_PR_TRIGGERED_SHA}"
+data='{"state":"'$status_state'","target_url":"'$BUILDKITE_BUILD_URL'","description":"'$description'","context":"buildkite/'$BUILDKITE_PIPELINE_SLUG'"}'
+
+echo "Setting buildkite/docs commit status to ${status_state}"
+curl -s -L \
+  -X POST \
+  -H "Accept: application/vnd.github+json" \
+  -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  "${githubPublishStatus}" \
+  -d "${data}"

.buildkite/scripts/build_pr_setup.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "The value is $MY_SECRET"
+
+state=$(buildkite-agent step get "state" --step "build-pr-setup")
+echo "State of build-pr-setup is $state"
+
+state=$(buildkite-agent step get "outcome" --step "build-pr-setup")
+echo "outcome of build-pr-setup is $state"
+
+state=$(buildkite-agent step get "state" --step "build-pr")
+echo "State is $state"
+
+state=$(buildkite-agent step get "outcome" --step "build-pr")
+echo "outcome of build-pr is $state"
+# Configure the git author and committer information
+export GIT_AUTHOR_NAME='Buildkite CI'
+export GIT_AUTHOR_EMAIL='[email protected]'
+export GIT_COMMITTER_NAME=$GIT_AUTHOR_NAME
+export GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL
+
+# The reminder of this hook should only be invoked for builds triggered by the Buildkite PR bot
+if [ -z ${GITHUB_PR_OWNER+set} ] || [ -z ${GITHUB_PR_REPO+set} ] || [ -z ${GITHUB_PR_TRIGGERED_SHA+set} ];then
+  exit 0
+fi
+
+gitHubToken=$(vault read -field=value secret/ci/elastic-docs/docs_preview_cleaner)
+
+githubPublishStatus="https://api.github.com/repos/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}/statuses/${GITHUB_PR_TRIGGERED_SHA}"
+data='{"state":"pending","target_url":"'$BUILDKITE_BUILD_URL'","description":"Build started.","context":"buildkite/'$BUILDKITE_PIPELINE_SLUG'"}'
+echo "Setting buildkite/docs commit status to pending"
+curl -s -L \
+  -X POST \
+  -H "Accept: application/vnd.github+json" \
+  -H "Authorization: Bearer ${gitHubToken}" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  "${githubPublishStatus}" \
+  -d "${data}"

resources/web/lib/prettify/lang-esql.js

@@ -28,7 +28,7 @@ PR['registerLangHandler'](
         ],
         [
          [PR['PR_COMMENT'], /^(?:\/\/[^\r\n]*|\/\*[\s\S]*?(?:\*\/|$))/],
-         [PR['PR_KEYWORD'], /^(?:ABS|ACOS|AND|APPEND_SEPARATOR|AS|ASC|ASIN|ATAN|ATAN2|AUTO_BUCKET|AVG|BY|CASE|CEIL|CIDR_MATCH|COALESCE|CONCAT|COS|COSH|COUNT|COUNT_DISTINCT|DATE_EXTRACT|DATE_FORMAT|DATE_PARSE|DATE_TRUNC|DESC|DISSECT|DROP|E|ENDS_WITH|ENRICH|EVAL|FIRST|FLOOR|FROM|GREATEST|GROK|IN|IS|IS_FINITE|IS_INFINITE|IS_NAN|LEFT|LENGTH|LOG10|LTRIM|MAX|MEDIAN|MEDIAN_ABSOLUTE_DEVIATION|MIN|KEEP|LAST|LEAST|LIKE|LIMIT|METADATA|MV_AVG|MV_CONCAT|MV_COUNT|MV_DEDUPE|MV_EXPAND|MV_MAX|MV_MEDIAN|MV_MIN|MV_SUM|NOT|NOW|NULL|NULLS|ON|OR|PERCENTILE|PI|POW|RENAME|REPLACE|RIGHT|RLIKE|ROUND|ROW|RTRIM|SHOW|SPLIT|SIN|SINH|SORT|SQRT|STARTS_WITH|STATS|SUBSTRING|TAN|TANH|TAU|TO_BOOL|TO_BOOLEAN|TO_DATETIME|TO_DBL|TO_DEGREES|TO_DOUBLE|TO_DT|TO_INT|TO_INTEGER|TO_IP|TO_LONG|TO_RADIANS|TO_STR|TO_STRING|TO_UL|TO_ULONG|TO_UNSIGNED_LONG|TO_VER|TO_VERSION|TRIM|SUM|WHERE|WITH)(?=[^\w-]|$)/i, null],
+         [PR['PR_KEYWORD'], /^(?:ABC|ABS|ACOS|AND|APPEND_SEPARATOR|AS|ASC|ASIN|ATAN|ATAN2|AUTO_BUCKET|AVG|BY|CASE|CEIL|CIDR_MATCH|COALESCE|CONCAT|COS|COSH|COUNT|COUNT_DISTINCT|DATE_EXTRACT|DATE_FORMAT|DATE_PARSE|DATE_TRUNC|DESC|DISSECT|DROP|E|ENDS_WITH|ENRICH|EVAL|FIRST|FLOOR|FROM|GREATEST|GROK|IN|IS|IS_FINITE|IS_INFINITE|IS_NAN|LEFT|LENGTH|LOG10|LTRIM|MAX|MEDIAN|MEDIAN_ABSOLUTE_DEVIATION|MIN|KEEP|LAST|LEAST|LIKE|LIMIT|METADATA|MV_AVG|MV_CONCAT|MV_COUNT|MV_DEDUPE|MV_EXPAND|MV_MAX|MV_MEDIAN|MV_MIN|MV_SUM|NOT|NOW|NULL|NULLS|ON|OR|PERCENTILE|PI|POW|RENAME|REPLACE|RIGHT|RLIKE|ROUND|ROW|RTRIM|SHOW|SPLIT|SIN|SINH|SORT|SQRT|STARTS_WITH|STATS|SUBSTRING|TAN|TANH|TAU|TO_BOOL|TO_BOOLEAN|TO_DATETIME|TO_DBL|TO_DEGREES|TO_DOUBLE|TO_DT|TO_INT|TO_INTEGER|TO_IP|TO_LONG|TO_RADIANS|TO_STR|TO_STRING|TO_UL|TO_ULONG|TO_UNSIGNED_LONG|TO_VER|TO_VERSION|TRIM|SUM|WHERE|WITH)(?=[^\w-]|$)/i, null],
          [PR['PR_LITERAL'], /^[+-]?(?:\.\d+|\d+(?:\.\d*)?)/i],
          [PR['PR_PLAIN'], /^[a-z_][\w-]*/i],
         ]),