[8.x] Add Gradle enterprise setup (#2307)

.buildkite/hooks/pre-command

@@ -46,3 +46,13 @@ if [[ "$USE_DRA_CREDENTIALS" == "true" ]]; then
   DRA_VAULT_ADDR=https://secrets.elastic.co:8200
   export DRA_VAULT_ADDR
 fi
+
+# Initialize the build scan and gobld annotations with empty/open <details> tags
+# This ensures that they are collapsible when they get appended to
+if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
+  cat << EOF | buildkite-agent annotate --context "gradle-build-scans" --style "info"
+<details>
+
+<summary>Gradle build scan links</summary>
+EOF
+fi

build.gradle

@@ -1,5 +1,13 @@
 import org.elasticsearch.hadoop.gradle.buildtools.ConcatFilesTask
+import java.lang.management.ManagementFactory;
+import java.time.LocalDateTime;
 
+import org.elasticsearch.gradle.Architecture
+import org.elasticsearch.gradle.OS
+import static org.elasticsearch.hadoop.gradle.util.CiUtils.safeName
+
+import java.lang.management.ManagementFactory
+import java.time.LocalDateTime
 description = 'Elasticsearch for Apache Hadoop'
 
 apply plugin: 'es.hadoop.build.root'
@@ -43,3 +51,117 @@ if (project.hasProperty("find-artifact")) {
         }
     }
 }
+
+
+
+// Resolving this early to avoid issues with the build scan plugin in combination with the configuration cache usage
+def taskNames = gradle.startParameter.taskNames.join(' ')
+
+develocity {
+
+  buildScan {
+
+    def onCI = System.getenv('CI') ? Boolean.parseBoolean(System.getenv('CI')) : false
+
+    // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
+    uploadInBackground = onCI == false
+
+    // Automatically publish scans from Elasticsearch CI
+    if (onCI) {
+      publishing.onlyIf { true }
+      server = 'https://gradle-enterprise.elastic.co'
+    } else if( server.isPresent() == false) {
+      publishing.onlyIf { false }
+    }
+
+    background {
+      tag OS.current().name()
+      tag Architecture.current().name()
+
+      if (onCI) { //Buildkite-specific build scan metadata
+        String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL')
+        def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH')
+        def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/
+        def repository = repoMatcher.matches() ? repoMatcher.group(2) : "<unknown>"
+        def jobLabel = System.getenv('BUILDKITE_LABEL') ?: ''
+        def jobName = safeName(jobLabel)
+
+        tag 'CI'
+        link 'CI Build', "${buildKiteUrl}#${System.getenv('BUILDKITE_JOB_ID')}"
+        value 'Job Number', System.getenv('BUILDKITE_BUILD_NUMBER')
+        value 'Build ID', System.getenv('BUILDKITE_BUILD_ID')
+        value 'Job ID', System.getenv('BUILDKITE_JOB_ID')
+
+        value 'Pipeline', System.getenv('BUILDKITE_PIPELINE_SLUG')
+        tag System.getenv('BUILDKITE_PIPELINE_SLUG')
+
+        value 'Job Name', jobName
+        tag jobName
+        if (jobLabel.contains("/")) {
+          jobLabel.split("/").collect { safeName(it) }.each { matrix ->
+            tag matrix
+          }
+        }
+
+        def uptime = ManagementFactory.getRuntimeMXBean().getUptime() / 1000;
+        def metricsStartTime = LocalDateTime.now().minusSeconds(uptime.longValue()).minusMinutes(15).toString()
+        def metricsEndTime = LocalDateTime.now().plusMinutes(15).toString()
+
+        link 'Agent Metrics',
+          "https://es-buildkite-agents.elastic.dev/app/metrics/detail/host/${System.getenv('BUILDKITE_AGENT_NAME')}?_a=(time:(from:%27${metricsStartTime}Z%27,interval:%3E%3D1m,to:%27${metricsEndTime}Z%27))"
+        link 'Agent Logs',
+          "https://es-buildkite-agents.elastic.dev/app/logs/stream?logFilter=(filters:!(),query:(language:kuery,query:%27host.name:%20${System.getenv('BUILDKITE_AGENT_NAME')}%27),timeRange:(from:%27${metricsStartTime}Z%27,to:%27${metricsEndTime}Z%27))"
+
+        if (branch) {
+          tag branch
+          value 'Git Branch', branch
+        }
+
+        // Add SCM information
+        def prId = System.getenv('BUILDKITE_PULL_REQUEST')
+        if (prId != 'false') {
+          def prBaseUrl = (System.getenv('BUILDKITE_PULL_REQUEST_REPO') - ".git").replaceFirst("git://", "https://")
+          value 'Git Commit ID', System.getenv('BUILDKITE_COMMIT')
+          tag "pr/${prId}"
+          tag 'pull-request'
+          link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}"
+          link 'Pull Request', "https://github.com/${repository}/pull/${prId}"
+        } else {
+          value 'Git Commit ID', System.getenv('BUILDKITE_COMMIT')
+          link 'Source', "https://github.com/${repository}/tree/${System.getenv('BUILDKITE_COMMIT')}"
+        }
+
+        buildFinished { result ->
+
+          buildScanPublished { scan
+            ->
+            // Attach build scan link as build metadata
+            // See: https://buildkite.com/docs/pipelines/build-meta-data
+            new ProcessBuilder('buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}")
+              .start()
+              .waitFor()
+
+            // Add a build annotation
+            // See: https://buildkite.com/docs/agent/v3/cli-annotate
+            def body = """<div class="mb3"><span class="p1 border rounded">${System.getenv('BUILDKITE_LABEL')}</span> :gradle: ${result.failures ? 'failed' : 'successful'} build: <a href="${scan.buildScanUri}"><code>gradle ${taskNames}</code></a></div>"""
+            def process = [
+              'buildkite-agent',
+              'annotate',
+              '--context',
+              result.failures ? 'gradle-build-scans-failed' : 'gradle-build-scans',
+              '--append',
+              '--style',
+              result.failures ? 'error' : 'info'
+            ].execute()
+            process.withWriter { it.write(body) }
+            // passing the body in as an argument has issues on Windows, so let's use stdin of the process instead
+            process.waitFor()
+          }
+        }
+      } else {
+        tag 'LOCAL'
+      }
+    }
+  }
+}
+

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/util/CiUtils.groovy

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.hadoop.gradle.util;
+
+public class CiUtils {
+
+    static String safeName(String input) {
+        return input.replaceAll("[^a-zA-Z0-9_\\-\\.]+", " ").trim().replaceAll(" ", "_").toLowerCase();
+    }
+
+}

settings.gradle

@@ -4,6 +4,10 @@ pluginManagement {
   }
 }
 
+plugins {
+  id "com.gradle.develocity" version "3.18.1"
+}
+
 rootProject.name = "elasticsearch-hadoop"
 
 include 'thirdparty'