[green-dragon-migration] Rewrite relay (#159)

## In This PR
Rewrite relay.groovy, namely the following changes were made:
* Take in a list of jobs rather than a pattern. We do this so that we
don't have to rely on `Jenkins.getInstance()`, `job.getName()`, or
`jenkins.getAllItems()` which are insecure Jenkins methods and will
result in security vulnerabilities if we approve them.
* Pull artifacts from S3 storage
* Implement a fix for LNT server overloading until we figure out issues
with the server


This code has been tested on https://green.lab.llvm.org/

Author: justice-adams-apple

zorg/jenkins/relay.groovy

@@ -1,43 +1,31 @@
 #!/usr/bin/env groovy
-@NonCPS
-private def get_matching_jobs(pattern) {
-    def jobs = []
-    for (job in Jenkins.getInstance().getAllItems(Job)) {
-        def jobname = job.getName()
-        def m = jobname =~ pattern
-        if (m) {
-            def shortname = m[0][1]
-            jobs.push([shortname, jobname])
-        }
-    }
-    return jobs
-}
-
 private def basename(path) {
     return path.drop(path.lastIndexOf('/') + 1)
 }
 
-private def relay_steps(job_pattern, artifact_url, last_good_properties_url) {
+private def relay_steps(joblist, artifact_url, last_good_properties_url) {
     // The upstream jobs triggering the relay produce a
     // "last_good_build.properties" file that contains a reference to the
     // compiler artifact that should be used for this run and which llvm
     // revision it is based on.
-    propfile = basename(last_good_properties_url)
-    sh """
-rm -f ${propfile}
-curl -fksSO "${last_good_properties_url}"
-"""
+    // Ensure you have the AWS CLI on path before triggering the relay
+    withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) {
+        propfile = basename(last_good_properties_url)
+        sh """
+          rm -f ${propfile}
+          aws s3 cp $S3_BUCKET/clangci/${last_good_properties_url} ${propfile}
+        """
+    }
+
     def props = readProperties file: propfile
-    def artifact = "http://green-dragon-21.local/artifacts/${props.ARTIFACT}"
+    def artifact = props.ARTIFACT
     currentBuild.setDisplayName("${props.GIT_DISTANCE}-${props.GIT_SHA}")
 
-    // Trigger all jobs with names matching the `job_pattern` regex.
-    def joblist = get_matching_jobs(job_pattern)
+    // Trigger all jobs within the provided list
     def parallel_builds = [:]
     for (j in joblist) {
-        def shortname = j[0]
-        def jobname = j[1]
-        parallel_builds[shortname] = {
+        def jobname = j
+        parallel_builds[jobname] = {
             def job_params = [
                 [$class: 'StringParameterValue',
                  name: 'ARTIFACT',
@@ -49,28 +37,39 @@ curl -fksSO "${last_good_properties_url}"
                  name: 'GIT_DISTANCE',
                  value: props.GIT_DISTANCE],
             ]
-            build job: jobname, parameters: job_params
+            build job: jobname, parameters: job_params, propagate: false
         }
     }
-    parallel parallel_builds
-}
 
-def pipeline(job_pattern,
-        artifact_url='http://green-dragon-21.local/artifacts/',
-        last_good_properties_url='http://green-dragon-21.local/artifacts/clang-stage1-RA/last_good_build.properties') {
-    node('master') {
-        stage('main') {
-            relay_steps job_pattern, artifact_url, last_good_properties_url
-        }
+    // Workaround to prevent LNT jobs from running in parallel and overloading the LNT server with submissions
+    if(joblist.any { it.contains("lnt-ctmark") }) {
+         for (j in joblist) {
+            parallel_builds[j].call()
+         }
+    } else {
+        parallel parallel_builds
     }
 }
 
-def lldb_pipeline(job_pattern,
-        artifact_url='http://green-dragon-21.local/artifacts/',
-        last_good_properties_url='http://green-dragon-21.local/artifacts/lldb-cmake/last_good_build.properties') {
-    node('master') {
+def pipeline(joblist,
+        artifact_url='llvm.org/clang-stage1-RA/latest',
+        last_good_properties_url='llvm.org/clang-stage1-RA/last_good_build.properties') {
+    node(label: 'macos-x86_64') {
         stage('main') {
-            relay_steps job_pattern, artifact_url, last_good_properties_url
+            // Download aws CLI used to gather artifacts
+            sh """
+              rm -rf venv
+              python3 -m venv venv
+              set +u
+              source ./venv/bin/activate
+              pip install awscli
+              set -u
+            """
+            withEnv([
+                "PATH=$PATH:$WORKSPACE/venv/bin:/usr/bin:/usr/local/bin"
+            ]) {
+                relay_steps joblist, artifact_url, last_good_properties_url
+            }
         }
     }
 }