Buildkite build PR job

.buildkite/build_pr_pipeline.yml

@@ -1,7 +1,23 @@
 steps:
-  - label: ":white_check_mark: Build docs PR"
-    command: |
-      bash .buildkite/scripts/build_pr.sh
+  - key: "build-pr-setup"
+    label: "setup"
+    command: ".buildkite/scripts/build_pr_commit_status.sh pending"
+  - key: "build-pr"
+    label: ":hammer: Build docs PR"
+    command: ".buildkite/scripts/build_pr.sh"
+    depends_on:
+      - step: build-pr-setup
+        allow_failure: true
     agents:
       provider: "gcp"
       image: family/docs-ubuntu-2204
+  - key: "teardown"
+    label: "teardown"
+    command: |
+      if [ $(buildkite-agent step get "outcome" --step "build-pr") == "passed" ]; then
+        .buildkite/scripts/build_pr_commit_status.sh success
+      else
+        .buildkite/scripts/build_pr_commit_status.sh failure
+      fi
+    depends_on:
+      - step: "build-pr"

.buildkite/hooks/pre-command

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euo pipefail
+
+function retry {
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
+
+# Secrets must be redacted
+# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "docs-build-pr" ]];then
+    export GITHUB_TOKEN=$(retry 5 vault kv get -field=value secret/ci/elastic-docs/docs_preview_cleaner)
+fi

.buildkite/scripts/build.sh

@@ -5,7 +5,7 @@ set +x
 
 # Configure the git author and committer information
 export GIT_AUTHOR_NAME='Buildkite CI'
-export GIT_AUTHOR_EMAIL='buildkite@elasticsearch-ci.elastic.co'
+export GIT_AUTHOR_EMAIL='docs-status+[email protected]'
 export GIT_COMMITTER_NAME=$GIT_AUTHOR_NAME
 export GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL
 

.buildkite/scripts/build_pr.sh

@@ -1,2 +1,99 @@
 #!/bin/bash
-echo "nothing to see yet"
+set -euo pipefail
+set +x
+
+# This script should only be invoked by the Buildkite PR bot
+if [ -z ${GITHUB_PR_BRANCH+set} ] || [ -z ${GITHUB_PR_TARGET_BRANCH+set} ] || [ -z ${GITHUB_PR_NUMBER+set} ] || [ -z ${GITHUB_PR_BASE_REPO+set} ];then
+  echo "One of the following env. variable GITHUB_PR_BRANCH, GITHUB_PR_TARGET_BRANCH, GITHUB_PR_NUMBER, GITHUB_PR_BASE_REPO is missing - exiting."
+  exit 1
+fi
+
+# Configure the git author and committer information
+export GIT_AUTHOR_NAME='Buildkite CI'
+export GIT_AUTHOR_EMAIL='[email protected]'
+export GIT_COMMITTER_NAME=$GIT_AUTHOR_NAME
+export GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL
+
+# Set some metadata for build filtering capabilities
+# https://buildkite.com/elastic/docs-build-pr/builds?meta_data[repo]=tech-content
+# https://buildkite.com/elastic/docs-build-pr/builds?meta_data[repo_pr]=tech-content_123
+buildkite-agent meta-data set "repo" "${GITHUB_PR_BASE_REPO}"
+buildkite-agent meta-data set "repo_pr" "${GITHUB_PR_BASE_REPO}_${GITHUB_PR_NUMBER}"
+
+rebuild_opt=""
+build_args=""
+TARGET_BRANCH=""
+
+# Define build docs arguments
+if [[ ${GITHUB_PR_COMMENT_VAR_REBUILD_OPT:="unset"} == "rebuild" ]];then
+  rebuild_opt=" --rebuild"
+elif [[ ${GITHUB_PR_COMMENT_VAR_SKIP_OPT:="unset"} == "skiplinkcheck" ]];then
+  build_args+=" --skiplinkcheck"
+  if [[ ${GITHUB_PR_COMMENT_VAR_WARN_OPT:="unset"} == "warnlinkcheck" ]];then
+    build_args+=" --warnlinkcheck"
+  fi
+fi
+
+buildkite-agent \
+    annotate \
+    --style "info" \
+    --context 'docs-info' \
+    "Triggered by a doc change in elastic/$GITHUB_PR_BASE_REPO PR: [#$GITHUB_PR_NUMBER](https://github.com/elastic/$GITHUB_PR_BASE_REPO/pull/$GITHUB_PR_NUMBER)"
+
+
+if [[ "${GITHUB_PR_BASE_REPO}" != 'docs' ]]; then
+  # Buildkite PR bot for repositories other than the `elastic/docs` repo are configured to
+  # always checkout the master branch of the `elastic/docs` repo (where the build logic resides).
+  # We first need to checkout the product repo / branch in a sub directory, that we'll reference
+  # in the build process.
+  echo "Cloning the ${GITHUB_PR_BASE_REPO} PR locally"
+
+  git clone --reference /opt/git-mirrors/elastic-$GITHUB_PR_BASE_REPO \
+    [email protected]:elastic/$GITHUB_PR_BASE_REPO.git ./product-repo
+
+  cd ./product-repo &&
+      git fetch origin pull/$GITHUB_PR_NUMBER/head:$GITHUB_PR_BRANCH &&
+      git switch $GITHUB_PR_BRANCH &&
+      cd ..
+
+  build_args+=" --sub_dir $GITHUB_PR_BASE_REPO:$GITHUB_PR_TARGET_BRANCH:./product-repo"
+else
+  # Buildkite PR bot for the `elastic/docs` repo is configured to checkout the PR directly into the workspace
+  # We don't have to do anything else in this case.
+
+  # Per https://github.com/elastic/docs/issues/1821, always rebuild all
+  # books for PRs to the docs repo, for now.
+  # When https://github.com/elastic/docs/issues/1823 is fixed, this
+  # should be removed and the original behavior restored.
+  rebuild_opt=" --rebuild"
+fi
+
+
+# Set the target branch and preview options
+TARGET_BRANCH="${GITHUB_PR_BASE_REPO}_bk_${GITHUB_PR_NUMBER}"
+PREVIEW_URL="https://${TARGET_BRANCH}.docs-preview.app.elstc.co"
+
+build_cmd="./build_docs --all --keep_hash \
+  --target_repo [email protected]:elastic/built-docs \
+  --reference /opt/git-mirrors/ \
+  --target_branch ${TARGET_BRANCH} \
+  --push \
+  --announce_preview ${PREVIEW_URL}/diff \
+  ${rebuild_opt} \
+  ${build_args}"
+
+echo "The following build command will be used"
+echo $build_cmd
+
+# Kick off the build
+ssh-agent bash -c "ssh-add && $build_cmd"
+
+buildkite-agent annotate \
+  --style "success" \
+  --context 'docs-info' \
+  --append \
+  "<br>Preview url: ${PREVIEW_URL}"
+
+buildkite-agent meta-data set pr_comment:doc-preview:head " * Documentation preview
+   - 📚 [HTML diff](${PREVIEW_URL}/diff)
+   - 📙 [Preview](${PREVIEW_URL})"

.buildkite/scripts/build_pr_commit_status.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# This hook should only be invoked for builds triggered by the Buildkite PR bot
+if [ -z ${GITHUB_PR_OWNER+set} ] || [ -z ${GITHUB_PR_REPO+set} ] || [ -z ${GITHUB_PR_TRIGGERED_SHA+set} ];then
+  exit 0
+fi
+
+status_state=$1
+description=''
+
+case $status_state in
+    pending)
+      description='Build started';;
+    success|failure|error)
+      description='Build finished';;
+    *)
+      echo "Invalid state $status_state"
+      exit 1;;
+esac
+
+githubPublishStatus="https://api.github.com/repos/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}/statuses/${GITHUB_PR_TRIGGERED_SHA}"
+data='{"state":"'$status_state'","target_url":"'$BUILDKITE_BUILD_URL'","description":"'$description'","context":"buildkite/'$BUILDKITE_PIPELINE_SLUG'"}'
+
+echo "Setting buildkite/${BUILDKITE_PIPELINE_SLUG} commit status to ${status_state}"
+curl -s -L \
+  -X POST \
+  -H "Accept: application/vnd.github+json" \
+  -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  "${githubPublishStatus}" \
+  -d "${data}"