Skip to content

Commit cf1a3d9

Browse files
ldionneldionne
committed
Implement the monolithic CI pipeline in the monorepo
This basically inlines the logic that was previously located in https://github.com/google/llvm-premerge-checks so it is part of the monorepo. This has the benefit of making it extremely easy for individual projects to understand and modify this logic for their own needs, unlike the current model where this logic lives in a separate non-LLVM repository. It also allows testing changes to the CI configuration using a simple Phabricator review, since the code that defines the CI pipeline is taken from the patch under review. This (or something equivalent) is necessary if we want to retain the current monolithic pre-commit CI throughout the GitHub PR transition. Since triggering the monolithic CI is currently attached to the system we use for triggering CI pipelines from Phabricator, we will lose that part of the CI when we move to GitHub PRs if we don't do anything. I've decided to rewrite the code as a shell script because the logic was fairly simple and it seemed a lot easier than figuring out how to pull only the relevant parts of llvm-premerge-checks into the monorepo. Furthermore, I think we should strive to move away from the monolithic CI altogether since sub-projects should understand, own and maintain the tests that are relevant for them to run in the CI (with LLVM providing the infrastructure). Hence, this is somewhat of a temporary solution until monolithic CI is removed entirely. Differential Revision: https://reviews.llvm.org/D158863
1 parent 3db5db9 commit cf1a3d9

File tree

4 files changed

+348
-6
lines changed

4 files changed

+348
-6
lines changed

‎.ci/generate-buildkite-pipeline-premerge

Lines changed: 210 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,6 @@
1414
# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
1515
#
1616

17-
if ! git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/|^clang/"; then
18-
# libcxx/, libcxxabi/, libunwind/, runtimes/, cmake/ or clang/ are not affected
19-
exit 0
20-
fi
21-
2217
reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
2318
if [[ "${reviewID}" != "" ]]; then
2419
buildMessage="https://llvm.org/${reviewID}"
@@ -51,3 +46,213 @@ if git diff --name-only HEAD~1 | grep -q -E "^clang/"; then
5146
branch: "${BUILDKITE_BRANCH}"
5247
EOF
5348
fi
49+
50+
#
51+
# If we're not running a more specific pipeline, generate a legacy monolithic pipeline
52+
# based on which subdirectories have been modified. We have some heuristics for this
53+
# to be reasonable.
54+
#
55+
# Individual projects should instead define the pre-commit CI tests that suits their
56+
# needs while letting them run on the infrastructure provided by LLVM.
57+
#
58+
function compute-projects-to-test() {
59+
projects=${@}
60+
for project in ${projects}; do
61+
echo "${project}"
62+
case ${project} in
63+
lld)
64+
for p in bolt cross-project-tests; do
65+
echo $p
66+
done
67+
;;
68+
llvm)
69+
for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do
70+
echo $p
71+
done
72+
;;
73+
clang)
74+
for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do
75+
echo $p
76+
done
77+
;;
78+
clang-tools-extra)
79+
echo libc
80+
;;
81+
mlir)
82+
echo flang
83+
;;
84+
*)
85+
# Nothing to do
86+
;;
87+
esac
88+
done
89+
}
90+
91+
function add-dependencies() {
92+
projects=${@}
93+
for project in ${projects}; do
94+
echo "${project}"
95+
case ${project} in
96+
bolt)
97+
for p in lld llvm; do
98+
echo $p
99+
done
100+
;;
101+
cross-project-tests)
102+
for p in lld clang; do
103+
echo $p
104+
done
105+
;;
106+
clang-tools-extra)
107+
for p in llvm clang; do
108+
echo $p
109+
done
110+
;;
111+
compiler-rt|libc|openmp)
112+
echo clang
113+
;;
114+
flang|lldb)
115+
for p in llvm clang; do
116+
echo $p
117+
done
118+
;;
119+
lld|mlir|polly)
120+
echo llvm
121+
;;
122+
*)
123+
# Nothing to do
124+
;;
125+
esac
126+
done
127+
}
128+
129+
function exclude-linux() {
130+
projects=${@}
131+
for project in ${projects}; do
132+
case ${project} in
133+
cross-project-tests) ;; # tests failing
134+
lldb) ;; # tests failing
135+
openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410
136+
*)
137+
echo "${project}"
138+
;;
139+
esac
140+
done
141+
}
142+
143+
function exclude-windows() {
144+
projects=${@}
145+
for project in ${projects}; do
146+
case ${project} in
147+
cross-project-tests) ;; # tests failing
148+
compiler-rt) ;; # tests taking too long
149+
openmp) ;; # TODO: having trouble with the Perl installation
150+
libc) ;; # no Windows support
151+
lldb) ;; # tests failing
152+
bolt) ;; # tests are not supported yet
153+
*)
154+
echo "${project}"
155+
;;
156+
esac
157+
done
158+
}
159+
160+
function keep-modified-projects() {
161+
projects=${@}
162+
git_diff="$(git diff --name-only HEAD~1)"
163+
for project in ${projects}; do
164+
if echo "${git_diff}" | grep -q -E "^${project}/"; then
165+
echo "${project}"
166+
fi
167+
done
168+
}
169+
170+
function check-targets() {
171+
projects=${@}
172+
for project in ${projects}; do
173+
case ${project} in
174+
clang-tools-extra)
175+
echo "check-clang-tools"
176+
;;
177+
compiler-rt)
178+
echo "check-all"
179+
;;
180+
cross-project-tests)
181+
echo "check-cross-project"
182+
;;
183+
lldb)
184+
echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
185+
;;
186+
pstl)
187+
echo "check-all"
188+
;;
189+
libclc)
190+
echo "check-all"
191+
;;
192+
*)
193+
echo "check-${project}"
194+
;;
195+
esac
196+
done
197+
}
198+
199+
# Figure out which projects need to be built on each platform
200+
all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
201+
modified_projects="$(keep-modified-projects ${all_projects})"
202+
203+
linux_projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
204+
linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
205+
linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
206+
207+
windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
208+
windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
209+
windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
210+
211+
# Generate the appropriate pipeline
212+
if [[ "${linux_projects}" != "" ]]; then
213+
cat <<EOF
214+
- label: ':linux: x64 Debian'
215+
artifact_paths:
216+
- '*_result.json'
217+
- 'build/monolithic-linux/test-results.xml'
218+
agents:
219+
queue: 'linux'
220+
retry:
221+
automatic:
222+
- exit_status: -1 # Agent was lost
223+
limit: 2
224+
- exit_status: 255
225+
limit: 2 # Forced agent shutdown
226+
timeout_in_minutes: 120
227+
env:
228+
CC: 'clang'
229+
CXX: 'clang++'
230+
commands:
231+
- './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"'
232+
EOF
233+
fi
234+
235+
if [[ "${windows_projects}" != "" ]]; then
236+
cat <<EOF
237+
- label: ':windows: x64 Windows'
238+
artifact_paths:
239+
- '*_result.json'
240+
- 'build/monolithic-windows/test-results.xml'
241+
agents:
242+
queue: 'windows'
243+
retry:
244+
automatic:
245+
- exit_status: -1 # Agent was lost
246+
limit: 2
247+
- exit_status: 255
248+
limit: 2 # Forced agent shutdown
249+
timeout_in_minutes: 150
250+
env:
251+
CC: 'cl'
252+
CXX: 'cl'
253+
LD: 'link'
254+
commands:
255+
- 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
256+
- 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
257+
EOF
258+
fi

‎.ci/generate-buildkite-pipeline-scheduled

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,43 @@ steps:
2727
message: "${BUILDKITE_MESSAGE}"
2828
commit: "${BUILDKITE_COMMIT}"
2929
branch: "${BUILDKITE_BRANCH}"
30-
EOF
30+
31+
- label: ':linux: x64 Debian'
32+
artifact_paths:
33+
- '*_result.json'
34+
- 'build/monolithic-linux/test-results.xml'
35+
agents:
36+
queue: 'linux'
37+
retry:
38+
automatic:
39+
- exit_status: -1 # Agent was lost
40+
limit: 2
41+
- exit_status: 255
42+
limit: 2 # Forced agent shutdown
43+
timeout_in_minutes: 120
44+
env:
45+
CC: 'clang'
46+
CXX: 'clang++'
47+
commands:
48+
- './.ci/monolithic-linux.sh "bolt;clang-tools-extra;compiler-rt;flang;libc;libclc;lld;llvm;mlir;polly;pstl" "check-all"'
49+
50+
- label: ':windows: x64 Windows'
51+
artifact_paths:
52+
- '*_result.json'
53+
- 'build/monolithic-windows/test-results.xml'
54+
agents:
55+
queue: 'windows'
56+
retry:
57+
automatic:
58+
- exit_status: -1 # Agent was lost
59+
limit: 2
60+
- exit_status: 255
61+
limit: 2 # Forced agent shutdown
62+
timeout_in_minutes: 150
63+
env:
64+
CC: 'cl'
65+
CXX: 'cl'
66+
LD: 'link'
67+
commands:
68+
- 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
69+
- 'bash .ci/monolithic-windows.sh "clang-tools-extra;flang;libclc;lld;llvm;mlir;polly;pstl" "check-all"'

‎.ci/monolithic-linux.sh

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env bash
2+
#===----------------------------------------------------------------------===##
3+
#
4+
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5+
# See https://llvm.org/LICENSE.txt for license information.
6+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7+
#
8+
#===----------------------------------------------------------------------===##
9+
10+
#
11+
# This script performs a monolithic build of the monorepo and runs the tests of
12+
# most projects on Linux. This should be replaced by per-project scripts that
13+
# run only the relevant tests.
14+
#
15+
16+
set -ex
17+
set -o pipefail
18+
19+
MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
20+
BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-linux}"
21+
22+
rm -rf ${BUILD_DIR}
23+
24+
ccache --zero-stats
25+
ccache --show-config
26+
function show-stats {
27+
ccache --print-stats
28+
}
29+
trap show-stats EXIT
30+
31+
projects="${1}"
32+
targets="${2}"
33+
34+
echo "--- cmake"
35+
pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt
36+
cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
37+
-D LLVM_ENABLE_PROJECTS="${projects}" \
38+
-G Ninja \
39+
-D CMAKE_BUILD_TYPE=Release \
40+
-D LLVM_ENABLE_ASSERTIONS=ON \
41+
-D LLVM_BUILD_EXAMPLES=ON \
42+
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
43+
-D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \
44+
-D LLVM_ENABLE_LLD=ON \
45+
-D CMAKE_CXX_FLAGS=-gmlt \
46+
-D BOLT_CLANG_EXE=/usr/bin/clang \
47+
-D LLVM_CCACHE_BUILD=ON
48+
49+
echo "--- ninja"
50+
ninja -C ${BUILD_DIR} ${targets}

‎.ci/monolithic-windows.sh

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/usr/bin/env bash
2+
#===----------------------------------------------------------------------===##
3+
#
4+
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5+
# See https://llvm.org/LICENSE.txt for license information.
6+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7+
#
8+
#===----------------------------------------------------------------------===##
9+
10+
#
11+
# This script performs a monolithic build of the monorepo and runs the tests of
12+
# most projects on Windows. This should be replaced by per-project scripts that
13+
# run only the relevant tests.
14+
#
15+
16+
set -ex
17+
set -o pipefail
18+
19+
MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
20+
BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-windows}"
21+
22+
rm -rf ${BUILD_DIR}
23+
24+
sccache --zero-stats
25+
function show-stats {
26+
sccache --show-stats
27+
}
28+
trap show-stats EXIT
29+
30+
projects="${1}"
31+
targets="${2}"
32+
33+
echo "--- cmake"
34+
pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt
35+
cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
36+
-D LLVM_ENABLE_PROJECTS="${projects}" \
37+
-G Ninja \
38+
-D CMAKE_BUILD_TYPE=Release \
39+
-D LLVM_ENABLE_ASSERTIONS=ON \
40+
-D LLVM_BUILD_EXAMPLES=ON \
41+
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
42+
-D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \
43+
-D COMPILER_RT_BUILD_ORC=OFF \
44+
-D CMAKE_C_COMPILER_LAUNCHER=sccache \
45+
-D CMAKE_CXX_COMPILER_LAUNCHER=sccache
46+
47+
echo "--- ninja"
48+
ninja -C ${BUILD_DIR} ${targets}

0 commit comments

Comments
 (0)