Skip to content

Commit ab4277b

Browse files
yxsamliuyxsamliu
authored
[HIP] Add blender test (#131)
Add test scripts to run blender with run-time compilation of HIP kernels, which can be used to test clang/llvm built by buildbot.
1 parent 610b959 commit ab4277b

File tree

7 files changed

+330
-0
lines changed

7 files changed

+330
-0
lines changed

‎External/HIP/CMakeLists.txt

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
include(External)
22
include(GPUTestVariant)
33
llvm_externals_find(TEST_SUITE_HIP_ROOT "hip" "HIP prerequisites")
4+
message(STATUS "TEST_SUITE_HIP_ROOT: ${TEST_SUITE_HIP_ROOT}")
5+
get_filename_component(HIP_CLANG_PATH ${CMAKE_CXX_COMPILER} DIRECTORY)
6+
message(STATUS "HIP_CLANG_PATH: ${HIP_CLANG_PATH}")
47

58
# Create targets for HIP tests that are part of the test suite.
69
macro(create_local_hip_tests VariantSuffix)
@@ -35,6 +38,17 @@ macro(create_local_hip_tests VariantSuffix)
3538
${VariantOffload} ${VariantSuffix}
3639
"${VariantCPPFLAGS}" "${VariantLibs}")
3740
endforeach()
41+
42+
# Add test for Blender.
43+
configure_file(workload/blender/test_blender.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test_blender.sh @ONLY)
44+
configure_file(workload/blender/verify_blender.sh.in ${CMAKE_CURRENT_BINARY_DIR}/verify_blender.sh @ONLY)
45+
file(COPY utils/log_data.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
46+
file(COPY utils/compare_image.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
47+
file(COPY utils/requirements.txt DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
48+
llvm_test_run(EXECUTABLE "/bin/bash" "test_blender.sh")
49+
llvm_test_verify(/bin/bash verify_blender.sh %o)
50+
llvm_add_test(blender.test test_blender.sh)
51+
list(APPEND VARIANT_SIMPLE_TEST_TARGETS blender.test)
3852
endmacro()
3953

4054
function(create_hip_test VariantSuffix)

‎External/HIP/lit.local.cfg

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,15 @@ import os
55
hip_env_vars = [
66
"HIP_VISIBLE_DEVICES",
77
"LD_LIBRARY_PATH",
8+
"HIP_BLENDER_TEST_CCC_OVERRIDE_OPTIONS",
9+
"HIP_BLENDER_TEST_PERF_THRESH",
10+
"HIP_BLENDER_TEST_OPTIONS",
11+
"HIP_BLENDER_TEST_BIN_DIR",
12+
"HIP_BLENDER_TEST_SCENES_DIR",
13+
"HIP_BLENDER_TEST_LOG_DIR",
14+
"HIP_BLENDER_TEST_WORK_DIR",
15+
"HIPCC_VERBOSE",
16+
"HIP_CLANG_PATH",
817
]
918

1019
for var in hip_env_vars:

‎External/HIP/utils/compare_image.py

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/bin/python3
2+
import argparse
3+
import os
4+
import sys
5+
6+
try:
7+
import cv2
8+
import numpy as np
9+
from skimage.metrics import structural_similarity as ssim
10+
except ImportError:
11+
print("One or more required packages are not installed. Please install them using the command:")
12+
print("pip install -r requirements.txt")
13+
sys.exit(1)
14+
15+
def mse(imageA, imageB):
16+
err = np.sum((imageA.astype("float") - imageB.astype("float")) ** 2)
17+
err /= float(imageA.shape[0] * imageA.shape[1] * imageA.shape[2])
18+
return err
19+
20+
def compare_images(image_path1, image_path2, ssim_threshold=0.9, mse_threshold=1000):
21+
image1 = cv2.imread(image_path1)
22+
image2 = cv2.imread(image_path2)
23+
24+
if image1 is None or image2 is None:
25+
raise ValueError("One or both of the image paths are invalid.")
26+
27+
if image1.shape != image2.shape:
28+
image2 = cv2.resize(image2, (image1.shape[1], image1.shape[0]))
29+
30+
smaller_side = min(image1.shape[:2])
31+
win_size = smaller_side if smaller_side % 2 == 1 else smaller_side - 1
32+
win_size = max(win_size, 3)
33+
34+
ssim_index, diff = ssim(image1, image2, multichannel=True, full=True, win_size=win_size, channel_axis=2)
35+
diff = (diff * 255).astype("uint8")
36+
37+
mse_value = mse(image1, image2)
38+
39+
# ssim_index is 'structural similarity index' (https://en.wikipedia.org/wiki/Structural_similarity_index_measure),
40+
# which is a popular measure of image similarity. Since it only measures gray-scale images, mse_value is also used,
41+
# which measures 'mean square error' (https://en.wikipedia.org/wiki/Mean_squared_error). Together they should be a
42+
# robust way to spot image differences without false alarms.
43+
if ssim_index < ssim_threshold or mse_value > mse_threshold:
44+
return -1, ssim_index, mse_value, diff
45+
else:
46+
return 0, ssim_index, mse_value, None
47+
48+
def main():
49+
parser = argparse.ArgumentParser(description="Compare two images for similarity.")
50+
parser.add_argument("--image", required=True, help="Path to the first image.")
51+
parser.add_argument("--ref", required=True, help="Path to the reference image.")
52+
parser.add_argument("--ssim-thresh", type=float, default=0.9, help="Threshold for the Structural Similarity Index (SSI).")
53+
parser.add_argument("--mse-thresh", type=float, default=1000, help="Threshold for the Mean Squared Error (MSE).")
54+
parser.add_argument("--quiet", action="store_true", help="Suppress output if specified.")
55+
56+
args = parser.parse_args()
57+
58+
result, ssim_index, mse_value, diff = compare_images(args.image, args.ref, args.ssim_thresh, args.mse_thresh)
59+
60+
if not args.quiet:
61+
print(f"Result: {result}")
62+
print(f"SSIM Index: {ssim_index:.3f}")
63+
print(f"MSE Value: {mse_value:.3g}")
64+
65+
if result == -1:
66+
image_path1_stem, image_path1_ext = os.path.splitext(args.image)
67+
diff_image_path = f"{image_path1_stem}_diff{image_path1_ext}"
68+
cv2.imwrite(diff_image_path, diff)
69+
if not args.quiet:
70+
print(f"Difference image saved to: {diff_image_path}")
71+
72+
if __name__ == "__main__":
73+
main()

‎External/HIP/utils/log_data.py

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/bin/python3
2+
import argparse
3+
import os
4+
import csv
5+
import time
6+
7+
def parse_arguments():
8+
parser = argparse.ArgumentParser(description='Record data and calculate statistics.')
9+
parser.add_argument('--data', type=float, required=True, help='The data value to record.')
10+
parser.add_argument('--log-file', type=str, required=True, help='The file to log data to.')
11+
parser.add_argument('--label', type=str, required=True, help='The label for the data.')
12+
parser.add_argument('--time-stamp', type=str, required=False, help='The timestamp for the data.')
13+
14+
args = parser.parse_args()
15+
return args
16+
17+
def read_existing_data(file_name):
18+
data = []
19+
if os.path.exists(file_name) and os.path.getsize(file_name) > 0:
20+
with open(file_name, 'r') as file:
21+
reader = csv.reader(file)
22+
for row in reader:
23+
if row and row[2].strip():
24+
try:
25+
data.append(float(row[2].strip()))
26+
except ValueError:
27+
continue
28+
return data
29+
30+
def calculate_average(data):
31+
if not data:
32+
return 0.0
33+
non_zero_data = [d for d in data if d != 0]
34+
if len(non_zero_data) == 0:
35+
return 0.0
36+
if len(non_zero_data) > 10:
37+
non_zero_data = non_zero_data[-10:]
38+
return sum(non_zero_data) / len(non_zero_data)
39+
40+
def calculate_percentage_difference(new_value, average):
41+
if average == 0:
42+
return 0.0
43+
return ((new_value - average) / average) * 100
44+
45+
def append_data(file_name, time_stamp, label, data):
46+
with open(file_name, 'a', newline='') as file:
47+
writer = csv.writer(file)
48+
writer.writerow([time_stamp, label, data])
49+
50+
def main():
51+
args = parse_arguments()
52+
53+
data = args.data
54+
log_file = args.log_file
55+
label = args.label
56+
time_stamp = args.time_stamp if args.time_stamp else time.strftime("%Y-%m-%d %H:%M:%S")
57+
58+
existing_data = read_existing_data(log_file)
59+
if not existing_data:
60+
average = data
61+
percentage_diff = 0.0
62+
else:
63+
average = calculate_average(existing_data)
64+
percentage_diff = calculate_percentage_difference(data, average)
65+
66+
append_data(log_file, time_stamp, label, data)
67+
68+
print(f"Average of the last 10 non-zero data points: {average:.3g}")
69+
print(f"Percentage difference from current data: {percentage_diff:.2f}%")
70+
71+
if __name__ == "__main__":
72+
main()

‎External/HIP/utils/requirements.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
opencv-python
2+
scikit-image
3+
numpy

‎External/HIP/workload/blender/test_blender.sh.in

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
#!/bin/bash
2+
3+
TEST_SUITE_HIP_ROOT=${TEST_SUITE_HIP_ROOT:-"@TEST_SUITE_HIP_ROOT@"}
4+
perf_thresh=${HIP_BLENDER_TEST_PERF_THRESH:-5}
5+
6+
export CCC_OVERRIDE_OPTIONS=${HIP_BLENDER_TEST_CCC_OVERRIDE_OPTIONS:-"+-v"}
7+
8+
export HIP_CLANG_PATH=${HIP_CLANG_PATH:-"@HIP_CLANG_PATH@"}
9+
export HIPCC_VERBOSE=${HIPCC_VERBOSE:-7}
10+
11+
blender_options=${HIP_BLENDER_TEST_OPTIONS:-"-F PNG --debug-cycles -- --cycles-device HIP"}
12+
blender_dir=${HIP_BLENDER_TEST_BIN_DIR:-"$TEST_SUITE_HIP_ROOT/blender"}
13+
scene_dir=${HIP_BLENDER_TEST_SCENES_DIR:-"$TEST_SUITE_HIP_ROOT/Blender_Scenes"}
14+
log_dir=${HIP_BLENDER_TEST_LOG_DIR:-"$scene_dir/logs"}
15+
work_dir=${HIP_BLENDER_TEST_WORK_DIR:-.}
16+
summary_file="summary.txt"
17+
18+
# Declare clang_hash as a global variable
19+
clang_hash=""
20+
21+
get_clang_hash() {
22+
clang_version_output=$($HIP_CLANG_PATH/clang -v 2>&1)
23+
clang_hash=$(echo "$clang_version_output" | grep -oP '(?<=llvm-project )\w{8}')
24+
echo "$clang_hash"
25+
}
26+
27+
get_blender_version() {
28+
blender_version_output=$($blender_dir/blender -v 2>&1)
29+
blender_version=$(echo "$blender_version_output" | grep -oP 'Blender \K[0-9]+\.[0-9]+')
30+
echo "$blender_version"
31+
}
32+
33+
# disable pre-built kernels and enable local kernel build
34+
check_and_rename_lib() {
35+
blender_version=$(get_blender_version)
36+
major_minor_version=${blender_version}
37+
lib_dir="$blender_dir/$major_minor_version/scripts/addons/cycles/lib"
38+
if [[ -d "$lib_dir" ]]; then
39+
mv "$lib_dir" "${lib_dir}.orig"
40+
fi
41+
}
42+
43+
log_kernel_compilation_time() {
44+
blender_output=$1
45+
kernel_compilation_time=$(sed -n 's/.*Kernel compilation finished in \([0-9]*\.[0-9]*\)s\./\1/p' $blender_output)
46+
echo "Collected kernel compilation time: $kernel_compilation_time s"
47+
if [[ ! -z "$kernel_compilation_time" ]]; then
48+
mkdir -p "$log_dir"
49+
kernel_log_file="$log_dir/kernel_compilation_time.log"
50+
python3 log_data.py --data "$kernel_compilation_time" --label "$clang_hash" --log-file "$kernel_log_file"
51+
fi
52+
}
53+
54+
render() {
55+
scene=$1
56+
out_file=${scene##*/}
57+
frame=${2:-1}
58+
out_file_full=${out_file}_$(printf "%03d" $frame).png
59+
input=$scene_dir/${scene}.blend
60+
output=$scene_dir/out/${out_file}_
61+
log_file="$log_dir/${out_file}.log"
62+
mkdir -p "$log_dir"
63+
echo "Render $input"
64+
65+
blender_output=$(mktemp)
66+
timeout 300 $blender_dir/blender -b $input -o ${output}### -f $frame $blender_options 2>&1 | tee $blender_output
67+
blender_return_code=${PIPESTATUS[0]}
68+
69+
average_time=$(grep -P "^\s*Path Tracing\s+\d+\.\d+\s+\d+\.\d+" $blender_output | awk '{print $4}')
70+
71+
log_kernel_compilation_time $blender_output
72+
73+
compare_output=$(mktemp)
74+
timeout 300 python3 compare_image.py --image $scene_dir/out/${out_file_full} --ref $scene_dir/ref/${out_file_full} 2>&1 | tee $compare_output
75+
compare_return_code=${PIPESTATUS[0]}
76+
77+
ssim=$(grep "SSIM Index:" $compare_output | awk '{print $3}')
78+
mse=$(grep "MSE Value:" $compare_output | awk '{print $3}')
79+
80+
previous_average=""
81+
percentage_difference=""
82+
perf_regress=0
83+
84+
if [[ ! -z "$average_time" && "$average_time" != "0" ]]; then
85+
log_output=$(python3 log_data.py --data "$average_time" --label "$clang_hash" --log-file "$log_file")
86+
87+
previous_average=$(echo "$log_output" | grep -oP '(?<=Average of the last 10 non-zero data points: )[^ ]+')
88+
percentage_difference=$(echo "$log_output" | grep -oP '(?<=Percentage difference from current data: )[^%]+')
89+
percentage_difference=${percentage_difference:-0}
90+
91+
if (( $(echo "$percentage_difference > $perf_thresh" | bc -l) )); then
92+
perf_regress=1
93+
fi
94+
fi
95+
96+
echo "$scene $frame $blender_return_code $compare_return_code $perf_regress $average_time $previous_average $percentage_difference $ssim $mse" >> $summary_file
97+
98+
if [[ $blender_return_code -ne 0 || $compare_return_code -ne 0 || $perf_regress -eq 1 ]]; then
99+
return 1
100+
fi
101+
return 0
102+
}
103+
104+
run() {
105+
cd $work_dir
106+
echo "Begin Blender test."
107+
hip_dir="$TEST_SUITE_HIP_ROOT"
108+
if [[ ! -e "$hip_dir" ]]; then
109+
echo "TEST_SUITE_HIP_ROOT=$TEST_SUITE_HIP_ROOT does not exist"
110+
exit -1
111+
fi
112+
echo "TEST_SUITE_HIP_ROOT=$TEST_SUITE_HIP_ROOT"
113+
if [[ ! -e "$blender_dir/blender" || ! -e "$scene_dir/scenes.txt" ]]; then
114+
echo "Skip HIP Blender test since no blender or test scenes found."
115+
echo "To set up HIP Blender test, download or build Blender from https://www.blender.org and install to External/hip/blender directory, and download Blender demo scenes and save to External/hip/Blender_scenes directory. Create a scenes.txt file under the Blender_scenes directory, with each line containing a scene file name and a frame number to render."
116+
exit -1
117+
fi
118+
119+
rm -rf ~/.cache/cycles
120+
echo "Scene Frame Blender_Return_Code Compare_Return_Code Perf_Regress Average_Time Previous_Average Percentage_Difference SSIM MSE" > $summary_file
121+
122+
check_and_rename_lib
123+
124+
clang_hash=$(get_clang_hash)
125+
126+
all_passed=true
127+
128+
while IFS=' ' read -r scene frame; do
129+
if [[ -z "$scene" || "$scene" == \#* ]]; then
130+
continue
131+
fi
132+
if ! render "$scene" "$frame"; then
133+
all_passed=false
134+
fi
135+
done < "$scene_dir/scenes.txt"
136+
137+
echo "HIP test summary:"
138+
cat $summary_file
139+
140+
if $all_passed; then
141+
echo "Blender test passes."
142+
else
143+
echo "Blender test fails."
144+
return 1
145+
fi
146+
}
147+
148+
run

‎External/HIP/workload/blender/verify_blender.sh.in

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
grep "Blender test passes" $1
4+
ret=$?
5+
if [[ $ret -ne 0 ]]; then
6+
cat $1
7+
fi
8+
if grep "Skip HIP Blender test since no blender or test scenes found" $1; then
9+
exit 0
10+
fi
11+
exit $ret

0 commit comments

Comments
 (0)