Skip to content

Commit 8bcdd1a

Browse files
justice-adams-applejustice-adams-apple
authored
[green-dragon-migration] Add stage 2 jobs (#171)
## In This PR * Update all existing stage 2 jobs * Add declarative pipeline for stage2 sanitizers All stage2 jobs have been updated to: * Run on the correct labels * upload artifacts to S3 * Set environment variables in `environment` closure when possible * Setup python3 venv with all dependencies to build/test This code has been tested on https://green.lab.llvm.org/
1 parent b839237 commit 8bcdd1a

File tree

3 files changed

+353
-45
lines changed

3 files changed

+353
-45
lines changed

‎zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto

Lines changed: 90 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,20 @@
11
pipeline {
2-
agent { label 'green-dragon-22' }
3-
4-
options { disableResume() }
2+
options {
3+
disableConcurrentBuilds()
4+
}
55

66
parameters {
7-
string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build')
8-
string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'description')
7+
string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on')
8+
9+
string(name: 'GIT_SHA', defaultValue: params.GIT_REVISION ?: '*/main', description: 'Git commit to build.')
10+
11+
string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Clang artifact to use')
12+
}
13+
14+
agent {
15+
node {
16+
label params.LABEL
17+
}
918
}
1019

1120
stages {
@@ -16,7 +25,7 @@ pipeline {
1625
[name: params.GIT_SHA]
1726
], extensions: [
1827
[$class: 'CloneOption',
19-
reference: '/Users/Shared/llvm-project.git']
28+
timeout: 30]
2029
], userRemoteConfigs: [
2130
[url: 'https://github.com/llvm/llvm-project.git']
2231
]])
@@ -33,71 +42,107 @@ pipeline {
3342
}
3443
}
3544
}
45+
stage('Setup Venv') {
46+
environment {
47+
PATH="$PATH:/usr/bin:/usr/local/bin"
48+
}
49+
steps {
50+
sh '''
51+
# Non-incremental, so always delete.
52+
rm -rf clang-build clang-install host-compiler *.tar.gz
53+
rm -rf venv
54+
python3 -m venv venv
55+
set +u
56+
source ./venv/bin/activate
57+
pip install -r ./llvm-zorg/zorg/jenkins/jobs/requirements.txt
58+
set -u
59+
'''
60+
}
61+
}
62+
stage('Fetch Artifact') {
63+
environment {
64+
PATH="$PATH:/usr/bin:/usr/local/bin"
65+
}
66+
steps {
67+
withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) {
68+
sh """
69+
source ./venv/bin/activate
70+
echo "ARTIFACT=${params.ARTIFACT}"
71+
python llvm-zorg/zorg/jenkins/monorepo_build.py fetch
72+
ls $WORKSPACE/host-compiler/lib/clang/
73+
VERSION=`ls $WORKSPACE/host-compiler/lib/clang/`
74+
"""
75+
}
76+
}
77+
}
3678
stage('Build') {
79+
environment {
80+
PATH="$PATH:/usr/bin:/usr/local/bin"
81+
}
3782
steps {
38-
timeout(600) {
39-
sh '''
40-
set -u
41-
rm -rf build.properties
83+
timeout(1200) {
84+
withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) {
85+
sh '''
86+
set -u
87+
rm -rf build.properties
88+
source ./venv/bin/activate
4289

43-
cd llvm-project
44-
git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true
90+
cd llvm-project
91+
git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true
4592

46-
git_desc=$(git describe --match "first_commit")
47-
export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-")
93+
git_desc=$(git describe --match "first_commit")
94+
export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-")
4895

49-
sha=$(echo ${git_desc} | cut -f 3 -d "-")
50-
export GIT_SHA=${sha:1}
96+
sha=$(echo ${git_desc} | cut -f 3 -d "-")
97+
export GIT_SHA=${sha:1}
5198

52-
cd -
99+
cd -
53100

54-
echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" > build.properties
101+
echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" > build.properties
55102

56-
export PATH=$PATH:/usr/bin:/usr/local/bin
57-
58-
rm -rf clang-build clang-install *.tar.gz
59-
60-
python llvm-zorg/zorg/jenkins/monorepo_build.py fetch
61-
python llvm-zorg/zorg/jenkins/monorepo_build.py clang build \
62-
--thinlto --projects="clang;compiler-rt" \
63-
--cmake-flag="-DCMAKE_DSYMUTIL=$WORKSPACE/host-compiler/bin/dsymutil"
64-
'''
103+
python llvm-zorg/zorg/jenkins/monorepo_build.py clang build \
104+
--thinlto --projects="clang;compiler-rt" \
105+
--cmake-flag="-DCMAKE_DSYMUTIL=$WORKSPACE/host-compiler/bin/dsymutil" \
106+
--runtimes="libunwind" \
107+
--cmake-flag="-DPython3_EXECUTABLE=$(which python)"
108+
'''
109+
}
65110
}
66111
}
67112
}
68-
69113
stage('Test') {
114+
environment {
115+
PATH="$PATH:/usr/bin:/usr/local/bin"
116+
}
70117
steps {
71-
timeout(240) {
118+
timeout(420) {
72119
sh '''
73120
set -u
74-
export PATH=$PATH:/usr/bin:/usr/local/bin
121+
source ./venv/bin/activate
75122

76123
rm -rf clang-build/testresults.xunit.xml
77124

78125
python llvm-zorg/zorg/jenkins/monorepo_build.py clang test
79126
'''
80127
}
81-
junit 'clang-build/**/testresults.xunit.xml'
128+
}
129+
post {
130+
always {
131+
script {
132+
junit "clang-build/**/testresults.xunit.xml"
133+
}
134+
}
82135
}
83136
}
84137
}
85138
post {
86139
always {
87-
scanForIssues tool: clang()
88-
}
89-
regression {
90-
emailext subject: '$DEFAULT_SUBJECT',
91-
presendScript: '$DEFAULT_PRESEND_SCRIPT',
92-
postsendScript: '$DEFAULT_POSTSEND_SCRIPT',
93-
recipientProviders: [
94-
[$class: 'CulpritsRecipientProvider'],
95-
[$class: 'DevelopersRecipientProvider'],
96-
[$class: 'RequesterRecipientProvider'],
97-
],
98-
replyTo: '$DEFAULT_REPLYTO',
99-
to: '$DEFAULT_RECIPIENTS',
100-
body:'$DEFAULT_CONTENT'
140+
script {
141+
// ToDo: Restore the issue scanner
142+
// scanForIssues tool: clang()
143+
144+
sh "rm -rf clang-build clang-install host-compiler *.tar.gz"
145+
}
101146
}
102147
}
103148
}

‎zorg/jenkins/jobs/jobs/clang-stage2-cmake-RgSan

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
pipeline {
2+
options {
3+
disableConcurrentBuilds()
4+
}
5+
6+
parameters {
7+
string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on')
8+
9+
string(name: 'GIT_SHA', defaultValue: params.GIT_REVISION ?: '*/main', description: 'Git commit to build.')
10+
11+
string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Clang artifact to use')
12+
}
13+
14+
agent {
15+
node {
16+
label params.LABEL
17+
}
18+
}
19+
20+
stages {
21+
stage('Checkout') {
22+
steps {
23+
dir('llvm-project') {
24+
checkout([$class: 'GitSCM', branches: [
25+
[name: params.GIT_SHA]
26+
], extensions: [
27+
[$class: 'CloneOption',
28+
timeout: 30]
29+
], userRemoteConfigs: [
30+
[url: 'https://github.com/llvm/llvm-project.git']
31+
]])
32+
}
33+
dir('llvm-zorg') {
34+
checkout([$class: 'GitSCM', branches: [
35+
[name: '*/main']
36+
], extensions: [
37+
[$class: 'CloneOption',
38+
reference: '/Users/Shared/llvm-zorg.git']
39+
], userRemoteConfigs: [
40+
[url: 'https://github.com/llvm/llvm-zorg.git']
41+
]])
42+
}
43+
}
44+
}
45+
stage('Setup Venv') {
46+
environment {
47+
PATH="$PATH:/usr/bin:/usr/local/bin"
48+
}
49+
steps {
50+
sh '''
51+
# Non-incremental, so always delete.
52+
rm -rf clang-build clang-install host-compiler *.tar.gz
53+
rm -rf venv
54+
python3 -m venv venv
55+
set +u
56+
source ./venv/bin/activate
57+
pip install -r ./llvm-zorg/zorg/jenkins/jobs/requirements.txt
58+
set -u
59+
'''
60+
}
61+
}
62+
stage('Fetch Artifact') {
63+
environment {
64+
PATH="$PATH:/usr/bin:/usr/local/bin"
65+
}
66+
steps {
67+
withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) {
68+
sh """
69+
source ./venv/bin/activate
70+
echo "ARTIFACT=${params.ARTIFACT}"
71+
python llvm-zorg/zorg/jenkins/monorepo_build.py fetch
72+
ls $WORKSPACE/host-compiler/lib/clang/
73+
VERSION=`ls $WORKSPACE/host-compiler/lib/clang/`
74+
"""
75+
}
76+
}
77+
}
78+
stage('Build') {
79+
environment {
80+
PATH="$PATH:/usr/bin:/usr/local/bin"
81+
HOST_INC_DIR="$WORKSPACE/host-compiler/bin/../include"
82+
}
83+
steps {
84+
withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) {
85+
sh '''
86+
source ./venv/bin/activate
87+
88+
cd llvm-project
89+
git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true
90+
91+
git_desc=$(git describe --match "first_commit")
92+
export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-")
93+
94+
sha=$(echo ${git_desc} | cut -f 3 -d "-")
95+
export GIT_SHA=${sha:1}
96+
97+
cd -
98+
99+
python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \
100+
--cmake-type=RelWithDebInfo \
101+
--projects="clang;clang-tools-extra" \
102+
--runtimes="libcxx;libcxxabi;libunwind" \
103+
--cmake-flag='-DLLVM_USE_SANITIZER=Address;Undefined' \
104+
--cmake-flag="-DLIBCXX_INCLUDE_TESTS=OFF" \
105+
--timeout=1800 \
106+
--cmake-flag="-DPython3_EXECUTABLE=$(which python)"
107+
'''
108+
}
109+
}
110+
}
111+
stage('Test') {
112+
environment {
113+
PATH="$PATH:/usr/bin:/usr/local/bin"
114+
ASAN_SYMBOLIZER_PATH="$WORKSPACE/host-compiler/bin/llvm-symbolizer"
115+
}
116+
steps {
117+
sh '''
118+
set -u
119+
source ./venv/bin/activate
120+
python llvm-zorg/zorg/jenkins/monorepo_build.py cmake test \
121+
--cmake-test-target=check-llvm \
122+
--cmake-test-target=check-clang
123+
'''
124+
}
125+
post {
126+
always {
127+
script {
128+
junit "clang-build/**/testresults.xunit.xml"
129+
}
130+
}
131+
}
132+
}
133+
}
134+
post {
135+
always {
136+
script {
137+
// ToDo: Restore the issue scanner
138+
// scanForIssues tool: clang()
139+
sh "rm -rf clang-build clang-install host-compiler *.tar.gz"
140+
}
141+
}
142+
}
143+
}
144+

0 commit comments

Comments
 (0)