Skip to content

Commit 51e9d02

Browse files
mofosynemofosyne
authored
Added a single test function script and fix debug-test.sh to be more robust (#7279)
* run-single-test.sh: added a single test function script and fix debug-test.sh to be more robust * debug-test.sh: combined execute and gdb test mode via -g flag * debug-test.sh: refactor * debug-test: refactor for clarity * debug-test.sh: comment style changes * debug-test.sh: fix gdb
1 parent d273c14 commit 51e9d02

File tree

2 files changed

+196
-94
lines changed

2 files changed

+196
-94
lines changed

‎docs/debugging-tests.md

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Debugging Tests Tips
22

3-
## How to run & debug a specific test without anything else to keep the feedback loop short?
3+
## How to run & execute or debug a specific test without anything else to keep the feedback loop short?
44

55
There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
66

@@ -10,13 +10,27 @@ For example, running the following command will output an interactive list from
1010

1111
It will then build & run in the debugger for you.
1212

13+
To just execute a test and get back a PASS or FAIL message run:
14+
1315
```bash
1416
./scripts/debug-test.sh test-tokenizer
17+
```
18+
19+
To test in GDB use the `-g` flag to enable gdb test mode.
20+
21+
```bash
22+
./scripts/debug-test.sh -g test-tokenizer
1523

1624
# Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows:
1725
>>> b main
1826
```
1927

28+
To speed up the testing loop, if you know your test number you can just run it similar to below:
29+
30+
```bash
31+
./scripts/debug-test.sh test 23
32+
```
33+
2034
For further reference use `debug-test.sh -h` to print help.
2135

2236
 
@@ -41,7 +55,7 @@ cmake -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON ..
4155
make -j
4256
```
4357

44-
#### Step 3.1: Identify Test Command for Debugging
58+
#### Step 3: Find all tests available that matches REGEX
4559

4660
The output of this command will give you the command & arguments needed to run GDB.
4761

@@ -69,11 +83,13 @@ Labels: main
6983
...
7084
```
7185

72-
So for test #1 we can tell these two pieces of relevant information:
86+
#### Step 4: Identify Test Command for Debugging
87+
88+
So for test #1 above we can tell these two pieces of relevant information:
7389
* Test Binary: `~/llama.cpp/build-ci-debug/bin/test-tokenizer-0`
7490
* Test GGUF Model: `~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf`
7591

76-
#### Step 3.2: Run GDB on test command
92+
#### Step 5: Run GDB on test command
7793

7894
Based on the ctest 'test command' report above we can then run a gdb session via this command below:
7995

‎scripts/debug-test.sh

Lines changed: 176 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -1,117 +1,203 @@
11
#!/bin/bash
2-
test_suite=${1:-}
3-
test_number=${2:-}
42

53
PROG=${0##*/}
64
build_dir="build-ci-debug"
75

8-
if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
9-
echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
10-
echo "Debug specific ctest program."
11-
echo
12-
echo "Options:"
13-
echo " -h, --help Display this help and exit"
14-
echo
15-
echo "Arguments:"
16-
echo " <test_regex> (Mandatory) Supply one regex to the script to filter tests"
17-
echo " (test_number) (Optional) Test number to run a specific test"
18-
echo
19-
echo "Example:"
20-
echo " $PROG test-tokenizer"
21-
echo " $PROG test-tokenizer 3"
22-
echo
23-
exit 0
24-
fi
6+
# Print Color Commands
7+
red=$(tput setaf 1)
8+
green=$(tput setaf 2)
9+
yellow=$(tput setaf 3)
10+
blue=$(tput setaf 4)
11+
magenta=$(tput setaf 5)
12+
cyan=$(tput setaf 6)
13+
normal=$(tput sgr0)
2514

26-
# Function to select and debug a test
27-
function select_test() {
28-
test_suite=${1:-test}
29-
test_number=${2:-}
30-
31-
# Sanity Check If Tests Is Detected
32-
printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
33-
tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
34-
if [ ${#tests[@]} -eq 0 ]
35-
then
36-
echo "No tests avaliable... check your compliation process..."
37-
echo "Exiting."
38-
exit 1
39-
fi
4015

41-
if [ -z $test_number ]
42-
then
43-
# List out avaliable tests
44-
printf "Which test would you like to debug?\n"
45-
id=0
46-
for s in "${tests[@]}"
47-
do
48-
echo "Test# ${id}"
49-
echo " $s"
50-
((id++))
51-
done
52-
53-
# Prompt user which test they wanted to run
54-
printf "\nRun test#? "
55-
read test_number
56-
else
57-
printf "\nUser Already Requested #${test_number}"
58-
fi
16+
# Print Help Message
17+
####################
5918

60-
# Start GDB with the requested test binary and arguments
61-
printf "Debugging(GDB) test: ${tests[test_number]}\n"
62-
# Change IFS (Internal Field Separator)
63-
sIFS=$IFS
64-
IFS=$'\n'
19+
print_full_help() {
20+
cat << EOF
21+
Usage: $PROG [OPTION]... <test_regex> (test_number)
22+
Debug specific ctest program.
6523
66-
# Get test args
67-
gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
68-
IFS=$sIFS
69-
printf "Debug arguments: ${gdb_args[test_number]}\n\n"
24+
Options:
25+
-h, --help display this help and exit
26+
-g run in gdb mode
7027
71-
# Expand paths if needed
72-
args=()
73-
for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\<//' -e 's/\>"//')
74-
do
75-
args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
76-
done
28+
Arguments:
29+
<test_regex> (Mandatory) Supply one regex to the script to filter tests
30+
(test_number) (Optional) Test number to run a specific test
7731
78-
# Execute debugger
79-
echo "gdb args: ${args[@]}"
80-
gdb --args ${args[@]}
32+
Example:
33+
$PROG test-tokenizer
34+
$PROG test-tokenizer 3
35+
EOF
8136
}
8237

38+
abort() {
39+
echo "Error: $1" >&2
40+
cat << EOF >&2
41+
Usage: $PROG [OPTION]... <test_regex> (test_number)
42+
Debug specific ctest program.
43+
Refer to --help for full instructions.
44+
EOF
45+
exit 1
46+
}
47+
48+
49+
# Dependency Sanity Check
50+
#########################
51+
52+
check_dependency() {
53+
command -v "$1" >/dev/null 2>&1 || {
54+
abort "$1 is required but not found. Please install it and try again."
55+
}
56+
}
57+
58+
check_dependency ctest
59+
check_dependency cmake
60+
61+
8362
# Step 0: Check the args
84-
if [ -z "$test_suite" ]
85-
then
86-
echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
87-
echo "Supply one regex to the script to filter tests,"
88-
echo "and optionally a test number to run a specific test."
89-
echo "Use --help flag for full instructions"
90-
exit 1
63+
########################
64+
65+
if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
66+
print_full_help >&2
67+
exit 0
68+
fi
69+
70+
# Parse command-line options
71+
gdb_mode=false
72+
while getopts "g" opt; do
73+
case $opt in
74+
g)
75+
gdb_mode=true
76+
echo "gdb_mode Mode Enabled"
77+
;;
78+
esac
79+
done
80+
81+
# Shift the option parameters
82+
shift $((OPTIND - 1))
83+
84+
# Positionial Argument Processing : <test_regex>
85+
if [ -z "${1}" ]; then
86+
abort "Test regex is required"
87+
else
88+
test_suite=${1:-}
9189
fi
9290

91+
# Positionial Argument Processing : (test_number)
92+
test_number=${2:-}
93+
94+
9395
# Step 1: Reset and Setup folder context
96+
########################################
97+
9498
## Sanity check that we are actually in a git repo
9599
repo_root=$(git rev-parse --show-toplevel)
96100
if [ ! -d "$repo_root" ]; then
97-
echo "Error: Not in a Git repository."
98-
exit 1
101+
abort "Not in a Git repository."
99102
fi
100103

101-
## Reset folder to root context of git repo
102-
pushd "$repo_root" || exit 1
104+
## Reset folder to root context of git repo and Create and enter build directory
105+
pushd "$repo_root"
106+
rm -rf "$build_dir" && mkdir "$build_dir" || abort "Failed to make $build_dir"
103107

104-
## Create and enter build directory
105-
rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
106108

107109
# Step 2: Setup Build Environment and Compile Test Binaries
108-
cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1
109-
pushd "$build_dir" && make -j || exit 1
110+
###########################################################
111+
112+
# Note: test-eval-callback requires -DLLAMA_CURL
113+
cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_CURL=1 || abort "Failed to build enviroment"
114+
pushd "$build_dir"
115+
make -j || abort "Failed to compile"
116+
popd > /dev/null || exit 1
117+
118+
119+
# Step 3: Find all tests available that matches REGEX
120+
####################################################
121+
122+
# Ctest Gather Tests
123+
# `-R test-tokenizer` : looks for all the test files named `test-tokenizer*` (R=Regex)
124+
# `-N` : "show-only" disables test execution & shows test commands that you can feed to GDB.
125+
# `-V` : Verbose Mode
126+
printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
127+
pushd "$build_dir"
128+
tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
129+
if [ ${#tests[@]} -eq 0 ]; then
130+
abort "No tests avaliable... check your compliation process..."
131+
fi
132+
popd > /dev/null || exit 1
133+
134+
135+
# Step 4: Identify Test Command for Debugging
136+
#############################################
137+
138+
# Select test number
139+
if [ -z $test_number ]; then
140+
# List out avaliable tests
141+
printf "Which test would you like to debug?\n"
142+
id=0
143+
for s in "${tests[@]}"
144+
do
145+
echo "Test# ${id}"
146+
echo " $s"
147+
((id++))
148+
done
149+
150+
# Prompt user which test they wanted to run
151+
printf "\nRun test#? "
152+
read test_number
153+
154+
else
155+
printf "\nUser Already Requested #${test_number}\n"
156+
157+
fi
158+
159+
# Grab all tests commands
160+
pushd "$build_dir"
161+
sIFS=$IFS # Save Initial IFS (Internal Field Separator)
162+
IFS=$'\n' # Change IFS (Internal Field Separator) (So we split ctest output by newline rather than by spaces)
163+
test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) # Get test args
164+
IFS=$sIFS # Reset IFS (Internal Field Separator)
165+
popd > /dev/null || exit 1
166+
167+
# Grab specific test command
168+
single_test_name="${tests[test_number]}"
169+
single_test_command="${test_args[test_number]}"
170+
171+
172+
# Step 5: Execute or GDB Debug
173+
##############################
174+
175+
printf "${magenta}Running Test #${test_number}: ${single_test_name}${normal}\n"
176+
printf "${cyan}single_test_command: ${single_test_command}${normal}\n"
110177

111-
# Step 3: Debug the Test
112-
select_test "$test_suite" "$test_number"
178+
if [ "$gdb_mode" = "true" ]; then
179+
# Execute debugger
180+
pushd "$repo_root" || exit 1
181+
eval "gdb --args ${single_test_command}"
182+
popd > /dev/null || exit 1
183+
184+
else
185+
# Execute Test
186+
pushd "$repo_root" || exit 1
187+
eval "${single_test_command}"
188+
exit_code=$?
189+
popd > /dev/null || exit 1
190+
191+
# Print Result
192+
printf "${blue}Ran Test #${test_number}: ${single_test_name}${normal}\n"
193+
printf "${yellow}Command: ${single_test_command}${normal}\n"
194+
if [ $exit_code -eq 0 ]; then
195+
printf "${green}TEST PASS${normal}\n"
196+
else
197+
printf "${red}TEST FAIL${normal}\n"
198+
fi
199+
200+
fi
113201

114-
# Step 4: Return to the directory from which the user ran the command.
115-
popd || exit 1
116-
popd || exit 1
117-
popd || exit 1
202+
# Return to the directory from which the user ran the command.
203+
popd > /dev/null || exit 1

0 commit comments

Comments
 (0)