Skip to content

Commit 64d855a

Browse files
author
Mikhail Gudim
committed
setup CI
1 parent 5192cb7 commit 64d855a

File tree

2 files changed

+262
-0
lines changed

2 files changed

+262
-0
lines changed

‎.gitlab-ci.yml

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# image should be available on the server
2+
image:
3+
name: gitlab.dc1.ventanamicro.com:5005/toolchain/llvm
4+
entrypoint: [""]
5+
6+
stages:
7+
- build
8+
- test
9+
- deploy
10+
11+
# TODO: split this up into several jobs. To do this,
12+
# need to figure out how to pass files from one job to another.
13+
build-test-benchmark:
14+
tags:
15+
- toolchain
16+
stage: build
17+
rules:
18+
- if: $CI_COMMIT_MESSAGE !=~ /Merge.+branch\s(.*)\sinto(.*)/
19+
script:
20+
- echo "Hello, $GITLAB_USER_LOGIN!"
21+
- whoami
22+
- pwd
23+
24+
# check that we can execute riscv64 with the plugin
25+
- cp $TEST_RISCV64 .
26+
- (export USE_QEMU_PLUGIN="1"; export QEMU_CPU="veyron-v2"; ./test-riscv64.elf)
27+
- ls
28+
29+
- export BRANCH_NAME=$CI_COMMIT_BRANCH
30+
# check that needed volumes are mounter correctly
31+
- export CCACHE_DIR=/mnt/ccache/llvm
32+
- export ARTIFACTS_DIR=/mnt/artifacts/llvm
33+
- ls $CCACHE_DIR
34+
- ls $ARTIFACTS_DIR
35+
36+
# check that ccache is actually working
37+
- ccache -s -v
38+
39+
- export STAGING_DIR=$ARTIFACTS_DIR/staging/$CI_COMMIT_BRANCH
40+
- rm -rf $STAGING_DIR
41+
- mkdir -p $STAGING_DIR
42+
43+
- cp -r /mnt/spec2017 $STAGING_DIR/
44+
- export SPEC_DIR=$STAGING_DIR/spec2017
45+
46+
- export BUILD_DIR=$STAGING_DIR/build
47+
- export INSTALL_DIR=$STAGING_DIR/install
48+
49+
- make prepare
50+
- make configure_llvm
51+
# TODO: run tests
52+
- make install_llvm
53+
54+
# run benchmarks with test workload
55+
- make clean_spec
56+
- make run_spec_test
57+
#- make check_spec_logs
58+
59+
# run benchmarks with train workload
60+
- make clean_spec
61+
- make run_spec_train
62+
- make check_spec_logs
63+
64+
# MKDIR_CP is defined by the docker container
65+
- $MKDIR_CP -listOfPaths $(ls $SPEC_DIR/cpu2017/benchspec/CPU/*/build/*/*.out) -pathPrefix $STAGING_DIR
66+
- $MKDIR_CP -listOfPaths $(ls $SPEC_DIR/cpu2017/benchspec/CPU/*/run/*/*.collect) -pathPrefix $STAGING_DIR
67+
- python3 $PARSE_BENCHMARK_DATA -pathToSpec $SPEC_DIR/cpu2017 -pathToOutput $STAGING_DIR/parsedBenchmarkData.json
68+
69+
# Baseline has to be first in the -listOfJsonFiles
70+
#- python3 $REPORT -listOfJsonFiles parsedBenchmarkData.json -dashBoardFile $ARTIFACTS_DIR/dashboard.csv | tee report.txt
71+
- python3 $REPORT -listOfJsonFiles $ARTIFACTS_DIR/commited/baseline/parsedBenchmarkData.json $STAGING_DIR/parsedBenchmarkData.json -dashBoardFile $ARTIFACTS_DIR/dashboard.csv | tee report.txt
72+
73+
74+
update-baseline:
75+
tags:
76+
- toolchain
77+
stage: deploy
78+
script:
79+
- export ARTIFACTS_DIR=/mnt/artifacts/llvm
80+
- export BRANCH_NAME=$(python3 $EXTRACT_BRANCH_NAME_FROM_COMMIT_MESSAGE -commitMessage "$CI_COMMIT_MESSAGE")
81+
- export STAGING_DIR=$ARTIFACTS_DIR/staging/$BRANCH_NAME
82+
- export INSTALL_DIR=$STAGING_DIR/install
83+
- make package_llvm
84+
- mv $STAGING_DIR/ventana-llvm.deb $ARTIFACTS_DIR/latest_build/ventana-llvm.deb
85+
- rm -rf $STAGING_DIR/build
86+
- rm -rf $STAGING_DIR/install
87+
- rm -rf $STAGING_DIR/spec2017
88+
89+
# GET_THIS_COMMIT_NUM is defined by the docker container
90+
- python3 $UPDATE_ARTIFACTS -artifactsDir $ARTIFACTS_DIR -branchToCommit=$(python3 $EXTRACT_BRANCH_NAME_FROM_COMMIT_MESSAGE -commitMessage "$CI_COMMIT_MESSAGE") -commitNo=$(python3 $GET_THIS_COMMIT_NUM -commitedDir $ARTIFACTS_DIR/commited)
91+
92+
only:
93+
variables:
94+
- $CI_COMMIT_MESSAGE =~ /Merge.+branch\s(.*)\sinto(.*)/
95+

‎Makefile

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
TOP = $(PWD)
2+
SHELL := /bin/bash
3+
4+
prepare:
5+
rm -rf $(BUILD_DIR)
6+
rm -rf $(INSTALL_DIR)
7+
mkdir -p $(BUILD_DIR)
8+
mkdir -p $(INSTALL_DIR)
9+
dpkg --extract /mnt/artifacts/gcc/latest_build/ventana-gcc.deb $(INSTALL_DIR)
10+
11+
configure_llvm:
12+
cd $(BUILD_DIR); \
13+
cmake $(TOP)/llvm \
14+
-G Ninja \
15+
-DCMAKE_BUILD_TYPE=Release \
16+
-DLLVM_ENABLE_ASSERTIONS=ON \
17+
-DCMAKE_C_COMPILER=$(CC) \
18+
-DCMAKE_CXX_COMPILER=$(CXX) \
19+
-DCMAKE_CXX_COMPILER_LAUNCHER="ccache" \
20+
-DCMAKE_CXX_FLAGS="-stdlib=libc++" \
21+
-DLLVM_USE_LINKER=lld \
22+
-DBUILD_SHARED_LIBS=ON \
23+
-DLLVM_TARGETS_TO_BUILD="RISCV" \
24+
-DLLVM_ENABLE_PROJECTS="clang;lld" \
25+
-DLLVM_OPTIMIZED_TABLEGEN=ON \
26+
-DLLVM_PARALLEL_LINK_JOBS=1 \
27+
-DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) \
28+
-DLLVM_BINUTILS_INCDIR=$(INSTALL_DIR)/x86_64-pc-linux-gnu/riscv64-linux-gnu/include
29+
30+
configure_llvm_native_riscv64_flang_build:
31+
rm -rf $(BUILD_DIR)
32+
mkdir -p $(BUILD_DIR)
33+
mkdir -p $(INSTALL_DIR)
34+
cd $(BUILD_DIR); \
35+
cmake $(TOP)/llvm \
36+
-G Ninja \
37+
-DCMAKE_BUILD_TYPE=Release \
38+
-DLLVM_ENABLE_ASSERTIONS=ON \
39+
-DLLVM_TARGETS_TO_BUILD="host" \
40+
-DLLVM_ENABLE_PROJECTS="clang;mlir;flang;openmp" \
41+
-DCMAKE_C_COMPILER=gcc \
42+
-DCMAKE_CXX_COMPILER=g++ \
43+
-DLLVM_PARALLEL_LINK_JOBS=1 \
44+
-DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) \
45+
-DLLVM_ENABLE_RUNTIMES="compiler-rt"
46+
47+
install_llvm:
48+
cd $(BUILD_DIR); cmake --build . --target install
49+
50+
package_llvm:
51+
mkdir -p $(INSTALL_DIR)/DEBIAN
52+
echo -e "\
53+
Package: ventanta-llvm \n\
54+
Version: 1.0 \n\
55+
Section: utils \n\
56+
Priority: optional \n\
57+
Architecture: all \n\
58+
Maintainer: Ventana Micro Systems \n\
59+
Description: LLVM, $(BRANCH_NAME) \n\
60+
" > $(INSTALL_DIR)/DEBIAN/control
61+
dpkg-deb --root-owner-group --build $(INSTALL_DIR)
62+
mv $(STAGING_DIR)/install.deb $(STAGING_DIR)/ventana-llvm.deb
63+
64+
MCPU=veyron-v1
65+
66+
67+
SPEC_OPTIMIZE_FLAGS="\
68+
--sysroot=$(INSTALL_DIR) \
69+
-mcpu=$(MCPU) \
70+
-O3 \
71+
-mllvm -riscv-enable-save-csr-in-ra=true \
72+
-mllvm -min-weight-ratio-needed-to-evict-hint=7.5 \
73+
-frecord-command-line \
74+
"
75+
SPEC_LD_FLAGS="\
76+
-fuse-ld=$(INSTALL_DIR)/riscv64-linux-gnu/bin/ld.bfd \
77+
-static \
78+
"
79+
# SPEC_DIR is defined in gitlab-ci.yml
80+
#
81+
define runSpecBenchmark
82+
cd $(SPEC_DIR)/cpu2017; \
83+
(\
84+
source shrc; \
85+
export USE_QEMU_PLUGIN="1"; \
86+
export QEMU_CPU="$(MCPU)"; \
87+
runcpu \
88+
--config=llvm-linux-riscv-ventana.cfg \
89+
--define label=$(BRANCH_NAME) \
90+
--define llvm_bin_dir="$(INSTALL_DIR)/bin" \
91+
--define optimize_flags=$(SPEC_OPTIMIZE_FLAGS) \
92+
--define ld_flags=$(SPEC_LD_FLAGS) \
93+
--action=validate \
94+
--size=$(2) \
95+
$(1) \
96+
)
97+
endef
98+
99+
100+
run_spec_test:
101+
$(call runSpecBenchmark,500.perlbench_r,test) & \
102+
$(call runSpecBenchmark,502.gcc_r,test) & \
103+
$(call runSpecBenchmark,505.mcf_r,test) & \
104+
$(call runSpecBenchmark,508.namd_r,test) & \
105+
$(call runSpecBenchmark,510.parest_r,test) & \
106+
$(call runSpecBenchmark,511.povray_r,test) & \
107+
$(call runSpecBenchmark,519.lbm_r,test) & \
108+
$(call runSpecBenchmark,520.omnetpp_r,test) & \
109+
$(call runSpecBenchmark,523.xalancbmk_r,test) & \
110+
$(call runSpecBenchmark,525.x264_r,test) & \
111+
$(call runSpecBenchmark,526.blender_r,test) & \
112+
$(call runSpecBenchmark,531.deepsjeng_r,test) & \
113+
$(call runSpecBenchmark,538.imagick_r,test) & \
114+
$(call runSpecBenchmark,541.leela_r,test) & \
115+
$(call runSpecBenchmark,544.nab_r,test) & \
116+
$(call runSpecBenchmark,557.xz_r,test) & \
117+
wait
118+
119+
run_spec_train:
120+
$(call runSpecBenchmark,500.perlbench_r,train) & \
121+
$(call runSpecBenchmark,502.gcc_r,train) & \
122+
$(call runSpecBenchmark,505.mcf_r,train) & \
123+
$(call runSpecBenchmark,508.namd_r,train) & \
124+
$(call runSpecBenchmark,510.parest_r,train) & \
125+
$(call runSpecBenchmark,511.povray_r,train) & \
126+
$(call runSpecBenchmark,519.lbm_r,train) & \
127+
$(call runSpecBenchmark,520.omnetpp_r,train) & \
128+
$(call runSpecBenchmark,523.xalancbmk_r,train) & \
129+
$(call runSpecBenchmark,525.x264_r,train) & \
130+
$(call runSpecBenchmark,526.blender_r,train) & \
131+
$(call runSpecBenchmark,531.deepsjeng_r,train) & \
132+
$(call runSpecBenchmark,538.imagick_r,train) & \
133+
$(call runSpecBenchmark,541.leela_r,train) & \
134+
$(call runSpecBenchmark,544.nab_r,train) & \
135+
$(call runSpecBenchmark,557.xz_r,train) & \
136+
wait
137+
138+
check_spec_logs:
139+
cd $(SPEC_DIR); \
140+
python3 spec.py \
141+
--specCPU2017Path=cpu2017 \
142+
--checkSpecLogs \
143+
--benchmarksList="\
144+
500.perlbench_r,\
145+
502.gcc_r,\
146+
505.mcf_r,\
147+
508.namd_r,\
148+
510.parest_r,\
149+
511.povray_r,\
150+
519.lbm_r,\
151+
520.omnetpp_r,\
152+
523.xalancbmk_r,\
153+
525.x264_r,\
154+
526.blender_r,\
155+
531.deepsjeng_r,\
156+
538.imagick_r,\
157+
541.leela_r,\
158+
544.nab_r,\
159+
557.xz_r\
160+
"
161+
162+
clean_spec:
163+
rm -rf $(SPEC_DIR)/cpu2017/benchspec/C*/*/run
164+
rm -rf $(SPEC_DIR)/cpu2017/benchspec/C*/*/build
165+
rm -rf $(SPEC_DIR)/cpu2017/benchspec/C*/*/exe
166+
rm -rf $(SPEC_DIR)/cpu2017/result/*
167+
rm -rf $(SPEC_DIR)/cpu2017/tmp/*

0 commit comments

Comments
 (0)