Skip to content

Commit db5f13a

Browse files
mikekgfbmalfet
mikekgfb
authored and
malfet
committed
Test fixes (#812)
* clean up unused files * fix tests: HF TOKEN not available on-pr, add evaluation.md to tests * markup docs * fix evaluations.md * add markup to native execution md * install wget for gguf.md testing, prevent evaluation.md failures * remove secrets from yml files * update * remove copy pasta from macosand macos-mps tests * typo * format
1 parent 7b473a0 commit db5f13a

File tree

10 files changed

+253
-134
lines changed

10 files changed

+253
-134
lines changed

‎.ci/scripts/run-docs

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,23 @@ if [ "$1" == "advanced" ]; then
7272
echo "*******************************************"
7373
bash -x ./run-advanced.sh
7474
echo "::endgroup::"
75-
echo "TBD"
7675
fi
7776

77+
if [ "$1" == "evaluation" ]; then
78+
79+
exit 0
80+
81+
echo "::group::Create script to run evaluation"
82+
python3 scripts/updown.py --file docs/evaluation.md --replace 'llama3:stories15M,-l 3:-l 2,meta-llama/Meta-Llama-3-8B:stories15M' --suppress huggingface-cli,HF_TOKEN > ./run-evaluation.sh
83+
# for good measure, if something happened to updown processor,
84+
# and it did not error out, fail with an exit 1
85+
echo "exit 1" >> ./run-evaluation.sh
86+
echo "::endgroup::"
87+
88+
echo "::group::Run evaluation"
89+
echo "*******************************************"
90+
cat ./run-evaluation.sh
91+
echo "*******************************************"
92+
bash -x ./run-evaluation.sh
93+
fi
94+

‎.github/workflows/run-readme-pr-macos.yml

Lines changed: 38 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -33,23 +33,16 @@ jobs:
3333
sysctl machdep.cpu.core_count
3434
echo "::endgroup::"
3535
36-
echo "::group::Create script to run README"
37-
python3 scripts/updown.py --file README.md --replace 'llama3:stories15M,-l 3:-l 2,meta-llama/Meta-Llama-3-8B-Instruct:stories15M' --suppress huggingface-cli,HF_TOKEN > ./run-readme.sh
38-
# for good measure, if something happened to updown processor,
39-
# and it did not error out, fail with an exit 1
40-
echo "exit 1" >> ./run-readme.sh
41-
echo "::endgroup::"
36+
.ci/scripts/run-docs readme
4237
43-
echo "::group::Run README"
44-
echo "*******************************************"
45-
cat ./run-readme.sh
38+
echo "::group::Completion"
39+
echo "tests complete"
4640
echo "*******************************************"
47-
bash -x ./run-readme.sh
4841
echo "::endgroup::"
49-
42+
5043

5144
test-quantization-macos:
52-
runs-on: macos-14-xlarge
45+
runs-on: macos-14-xlarge
5346
steps:
5447
- name: Checkout code
5548
uses: actions/checkout@v2
@@ -75,19 +68,7 @@ jobs:
7568
sysctl machdep.cpu.core_count
7669
echo "::endgroup::"
7770
78-
echo "::group::Create script to run quantization"
79-
python3 scripts/updown.py --file docs/quantization.md --replace 'llama3:stories15M,-l 3:-l 2,meta-llama/Meta-Llama-3-8B-Instruct:stories15M' --suppress huggingface-cli,HF_TOKEN > ./run-quantization.sh
80-
# for good measure, if something happened to updown processor,
81-
# and it did not error out, fail with an exit 1
82-
echo "exit 1" >> ./run-quantization.sh
83-
echo "::endgroup::"
84-
85-
echo "::group::Run quantization"
86-
echo "*******************************************"
87-
cat ./run-quantization.sh
88-
echo "*******************************************"
89-
bash -x ./run-quantization.sh
90-
echo "::endgroup::"
71+
.ci/scripts/run-docs quantization
9172
9273
echo "::group::Completion"
9374
echo "tests complete"
@@ -97,7 +78,6 @@ jobs:
9778

9879
test-gguf-macos:
9980
runs-on: macos-14-xlarge
100-
secrets: inherit
10181
steps:
10282
- name: Checkout code
10383
uses: actions/checkout@v2
@@ -110,7 +90,6 @@ jobs:
11090
with:
11191
xcode-version: '15.3'
11292
- name: Run script
113-
secrets-env: "HF_TOKEN_PERIODIC"
11493
run: |
11594
set -x
11695
# NS: Remove previous installation of torch first
@@ -124,25 +103,42 @@ jobs:
124103
sysctl machdep.cpu.core_count
125104
echo "::endgroup::"
126105
127-
# echo "::group::Install newer objcopy that supports --set-section-alignment"
128-
# yum install -y devtoolset-10-binutils
129-
# export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH
130-
# echo "::endgroup::"
131-
132-
echo "::group::Create script to run gguf"
133-
python3 scripts/updown.py --file docs/GGUF.md > ./run-gguf.sh
134-
# for good measure, if something happened to updown processor,
135-
# and it did not error out, fail with an exit 1
136-
echo "exit 1" >> ./run-gguf.sh
137-
echo "::endgroup::"
106+
.ci/scripts/run-docs gguf
138107
139-
echo "::group::Run gguf"
140-
echo "*******************************************"
141-
cat ./run-gguf.sh
108+
echo "::group::Completion"
109+
echo "tests complete"
142110
echo "*******************************************"
143-
bash -x ./run-gguf.sh
144111
echo "::endgroup::"
145112
113+
test-advanced-macos:
114+
runs-on: macos-14-xlarge
115+
steps:
116+
- name: Checkout code
117+
uses: actions/checkout@v2
118+
- uses: actions/setup-python@v4
119+
with:
120+
python-version: '3.10.11'
121+
- name: Setup Xcode
122+
if: runner.os == 'macOS'
123+
uses: maxim-lobanov/setup-xcode@v1
124+
with:
125+
xcode-version: '15.3'
126+
- name: Run script
127+
run: |
128+
set -x
129+
# NS: Remove previous installation of torch first
130+
# as this script does not isntall anything into conda env but rather as system dep
131+
pip3 uninstall -y torch || true
132+
set -eou pipefail
133+
134+
echo "::group::Print machine info"
135+
uname -a
136+
sysctl machdep.cpu.brand_string
137+
sysctl machdep.cpu.core_count
138+
echo "::endgroup::"
139+
140+
.ci/scripts/run-docs advanced
141+
146142
echo "::group::Completion"
147143
echo "tests complete"
148144
echo "*******************************************"

‎.github/workflows/run-readme-pr-mps.yml

Lines changed: 59 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -25,29 +25,17 @@ jobs:
2525
sysctl machdep.cpu.core_count
2626
echo "::endgroup::"
2727
28-
# echo "::group::Install newer objcopy that supports --set-section-alignment"
29-
# yum install -y devtoolset-10-binutils
30-
# export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH
31-
# echo "::endgroup::"
32-
33-
echo "::group::Create script to run README"
34-
python3 scripts/updown.py --file README.md --replace 'llama3:stories15M,-l 3:-l 2,meta-llama/Meta-Llama-3-8B-Instruct:stories15M' --suppress huggingface-cli,HF_TOKEN > ./run-readme.sh
35-
# for good measure, if something happened to updown processor,
36-
# and it did not error out, fail with an exit 1
37-
echo "exit 1" >> ./run-readme.sh
38-
echo "::endgroup::"
39-
40-
echo "::group::Run README"
41-
echo "*******************************************"
42-
cat ./run-readme.sh
28+
.ci/scripts/run-docs readme
29+
30+
echo "::group::Completion"
31+
echo "tests complete"
4332
echo "*******************************************"
44-
bash -x ./run-readme.sh
4533
echo "::endgroup::"
46-
34+
4735
test-quantization-mps-macos:
4836
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
4937
with:
50-
runner: macos-m1-stable # neeps MPS, was macos-m1-stable
38+
runner: macos-m1-14
5139
script: |
5240
set -x
5341
conda create -y -n test-quantization-mps-macos python=3.10.11
@@ -64,25 +52,62 @@ jobs:
6452
sysctl machdep.cpu.core_count
6553
echo "::endgroup::"
6654
67-
# echo "::group::Install newer objcopy that supports --set-section-algnment"
68-
# yum install -y devtoolset-10-binutils
69-
# export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH
70-
# echo "::endgroup::"
71-
72-
echo "::group::Create script to run quantization"
73-
python3 scripts/updown.py --file docs/quantization.md --replace llama3:stories15M --suppress huggingface-cli,HF_TOKEN > ./run-quantization.sh
74-
# for good measure, if something happened to updown processor,
75-
# and it did not error out, fail with an exit 1
76-
echo "exit 1" >> ./run-quantization.sh
77-
echo "::endgroup::"
78-
79-
echo "::group::Run quantization"
55+
.ci/scripts/run-docs quantization
56+
57+
echo "::group::Completion"
58+
echo "tests complete"
8059
echo "*******************************************"
81-
cat ./run-quantization.sh
60+
echo "::endgroup::"
61+
62+
test-gguf-mps-macos:
63+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
64+
with:
65+
runner: macos-m1-14 # neeps MPS, was macos-m1-stable
66+
script: |
67+
set -x
68+
conda create -y -n test-quantization-mps-macos python=3.10.11
69+
conda activate test-quantization-mps-macos
70+
# NS: Remove previous installation of torch first
71+
# as this script does not isntall anything into conda env
72+
# but rather system dep
73+
pip3 uninstall -y torch || true
74+
set -eou pipefail
75+
76+
echo "::group::Print machine info"
77+
uname -a
78+
sysctl machdep.cpu.brand_string
79+
sysctl machdep.cpu.core_count
80+
echo "::endgroup::"
81+
82+
.ci/scripts/run-docs gguf
83+
84+
echo "::group::Completion"
85+
echo "tests complete"
8286
echo "*******************************************"
83-
bash -x ./run-quantization.sh
8487
echo "::endgroup::"
85-
88+
89+
test-advanced-mps-macos:
90+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
91+
with:
92+
runner: macos-m1-14 # neeps MPS, was macos-m1-stable
93+
script: |
94+
set -x
95+
conda create -y -n test-quantization-mps-macos python=3.10.11
96+
conda activate test-quantization-mps-macos
97+
# NS: Remove previous installation of torch first
98+
# as this script does not isntall anything into conda env
99+
# but rather system dep
100+
pip3 uninstall -y torch || true
101+
set -eou pipefail
102+
103+
echo "::group::Print machine info"
104+
uname -a
105+
sysctl machdep.cpu.brand_string
106+
sysctl machdep.cpu.core_count
107+
echo "::endgroup::"
108+
109+
.ci/scripts/run-docs advanced
110+
86111
echo "::group::Completion"
87112
echo "tests complete"
88113
echo "*******************************************"

‎.github/workflows/run-readme-pr.yml

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,8 @@ on:
1010
jobs:
1111
test-readme-any:
1212
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
13-
secrets: inherit
1413
with:
1514
runner: linux.g5.4xlarge.nvidia.gpu
16-
secrets-env: "HF_TOKEN_PERIODIC"
1715
gpu-arch-type: cuda
1816
gpu-arch-version: "12.1"
1917
timeout: 60
@@ -38,7 +36,6 @@ jobs:
3836
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
3937
with:
4038
runner: linux.g5.4xlarge.nvidia.gpu
41-
secrets-env: "HF_TOKEN_PERIODIC"
4239
gpu-arch-type: cuda
4340
gpu-arch-version: "12.1"
4441
timeout: 60
@@ -104,10 +101,8 @@ jobs:
104101
105102
test-gguf-any:
106103
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
107-
secrets: inherit
108104
with:
109105
runner: linux.g5.4xlarge.nvidia.gpu
110-
secrets-env: "HF_TOKEN_PERIODIC"
111106
gpu-arch-type: cuda
112107
gpu-arch-version: "12.1"
113108
timeout: 60
@@ -130,10 +125,8 @@ jobs:
130125
131126
test-gguf-cpu:
132127
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
133-
secrets: inherit
134128
with:
135129
runner: linux.g5.4xlarge.nvidia.gpu
136-
secrets-env: "HF_TOKEN_PERIODIC"
137130
gpu-arch-type: cuda
138131
gpu-arch-version: "12.1"
139132
timeout: 60
@@ -157,10 +150,8 @@ jobs:
157150
158151
test-advanced-any:
159152
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
160-
secrets: inherit
161153
with:
162154
runner: linux.g5.4xlarge.nvidia.gpu
163-
secrets-env: "HF_TOKEN_PERIODIC"
164155
gpu-arch-type: cuda
165156
gpu-arch-version: "12.1"
166157
timeout: 60
@@ -184,10 +175,8 @@ jobs:
184175
185176
test-advanced-cpu:
186177
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
187-
secrets: inherit
188178
with:
189179
runner: linux.g5.4xlarge.nvidia.gpu
190-
secrets-env: "HF_TOKEN_PERIODIC"
191180
gpu-arch-type: cuda
192181
gpu-arch-version: "12.1"
193182
timeout: 60
@@ -208,12 +197,10 @@ jobs:
208197
echo "*******************************************"
209198
echo "::endgroup::"
210199
211-
test-torchtune-any:
200+
test-evaluation-any:
212201
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
213-
secrets: inherit
214202
with:
215203
runner: linux.g5.4xlarge.nvidia.gpu
216-
secrets-env: "HF_TOKEN_PERIODIC"
217204
gpu-arch-type: cuda
218205
gpu-arch-version: "12.1"
219206
timeout: 60
@@ -227,18 +214,31 @@ jobs:
227214
export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH
228215
echo "::endgroup::"
229216
230-
echo "::group::Create script to run torchtune"
231-
python3 scripts/updown.py --file docs/torchtune.md --replace 'llama3:stories15M,-l 3:-l 2,meta-llama/Meta-Llama-3-8B:stories15M' --suppress huggingface-cli,HF_TOKEN > ./run-torchtune.sh
232-
# for good measure, if something happened to updown processor,
233-
# and it did not error out, fail with an exit 1
234-
echo "exit 1" >> ./run-torchtune.sh
235-
echo "::endgroup::"
217+
.ci/scripts/run-docs evaluation
236218
237-
echo "::group::Run advanced"
238-
echo "*******************************************"
239-
cat ./run-torchtune.sh
219+
echo "::group::Completion"
220+
echo "tests complete"
240221
echo "*******************************************"
241-
bash -x ./run-torchtune.sh
222+
echo "::endgroup::"
223+
224+
test-evaluation-cpu:
225+
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
226+
with:
227+
runner: linux.g5.4xlarge.nvidia.gpu
228+
gpu-arch-type: cuda
229+
gpu-arch-version: "12.1"
230+
timeout: 60
231+
script: |
232+
echo "::group::Print machine info"
233+
uname -a
234+
echo "::endgroup::"
235+
236+
echo "::group::Install newer objcopy that supports --set-section-alignment"
237+
yum install -y devtoolset-10-binutils
238+
export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH
239+
echo "::endgroup::"
240+
241+
TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs evaluation
242242
243243
echo "::group::Completion"
244244
echo "tests complete"

‎docs/ADVANCED-USERS.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ Torchchat is currently in a pre-release state and under extensive development.
1010

1111
[shell default]: HF_TOKEN="${SECRET_HF_TOKEN_PERIODIC}" huggingface-cli login
1212

13+
[shell default]: ./install_requirements.sh
14+
1315
[shell default]: TORCHCHAT_ROOT=${PWD} ./scripts/install_et.sh
1416

1517

0 commit comments

Comments
 (0)