Skip to content

Commit 82fd241

Browse files
mikekgfbmalfet
mikekgfb
authored and
malfet
committed
Dtype tests (#147)
* tiktoken integration, part 1 * update tests * dtype tests
1 parent 0044739 commit 82fd241

File tree

1 file changed

+75
-72
lines changed

1 file changed

+75
-72
lines changed

‎.github/workflows/compile-bf16.yml

Lines changed: 75 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -43,75 +43,78 @@ jobs:
4343
export MODEL_PATH=checkpoints/stories15M/stories15M.pt
4444
export MODEL_NAME=stories15M
4545
export MODEL_DIR=/tmp
46-
export DTYPE=bfloat16
47-
if [ $(uname -s) == Darwin ]; then
48-
export DTYPE=float16
49-
fi
50-
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
51-
cat ./output_eager
52-
python generate.py --dtype ${DTYPE} --compile --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
53-
cat ./output_compiled
54-
python export.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
55-
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
56-
cat ./output_aoti
57-
58-
echo "******************************************"
59-
echo "******* Emb: channel-wise quantized ******"
60-
echo "******************************************"
61-
python generate.py --dtype ${DTYPE} --quant '{"embedding" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
62-
cat ./output_eager
63-
python generate.py --dtype ${DTYPE} --compile --quant '{"embedding" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
64-
cat ./output_compiled
65-
python export.py --dtype ${DTYPE} --quant '{"embedding" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
66-
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
67-
cat ./output_aoti
68-
69-
echo "******************************************"
70-
echo "******** Emb: group-wise quantized *******"
71-
echo "******************************************"
72-
python generate.py --dtype ${DTYPE} --quant '{"embedding" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
73-
cat ./output_eager
74-
python generate.py --dtype ${DTYPE} --compile --quant '{"embedding" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
75-
cat ./output_compiled
76-
python export.py --dtype ${DTYPE} --quant '{"embedding" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
77-
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
78-
cat ./output_aoti
79-
80-
echo "******************************************"
81-
echo "******* INT8 channel-wise quantized ******"
82-
echo "******************************************"
83-
python generate.py --dtype ${DTYPE} --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
84-
cat ./output_eager
85-
python generate.py --dtype ${DTYPE} --compile --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
86-
cat ./output_compiled
87-
python export.py --dtype ${DTYPE} --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
88-
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
89-
cat ./output_aoti
90-
91-
echo "******************************************"
92-
echo "******** INT8 group-wise quantized *******"
93-
echo "******************************************"
94-
if [ ${DTYPE} == float16 ]; then
95-
DTYPE=bfloat16
96-
fi
97-
98-
python generate.py --dtype ${DTYPE} --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
99-
cat ./output_eager
100-
python generate.py --dtype ${DTYPE} --compile --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
101-
cat ./output_compiled
102-
python export.py --dtype ${DTYPE} --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
103-
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
104-
cat ./output_aoti
105-
106-
echo "******************************************"
107-
echo "******** INT4 group-wise quantized *******"
108-
echo "******************************************"
109-
python generate.py --dtype ${DTYPE} --quant '{"linear:int4" : {"group_size": 32}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
110-
cat ./output_eager
111-
python generate.py --dtype ${DTYPE} --compile --quant '{"linear:int4" : {"group_size": 32}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
112-
cat ./output_compiled
113-
python export.py --dtype ${DTYPE} --quant '{"linear:int4" : {"group_size": 32}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
114-
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
115-
cat ./output_aoti
116-
117-
echo "tests complete"
46+
for DTYPE in bfloat16 float16 float32; do
47+
if [ $(uname -s) == Darwin ]; then
48+
export DTYPE=float16
49+
fi
50+
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
51+
cat ./output_eager
52+
python generate.py --dtype ${DTYPE} --compile --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
53+
cat ./output_compiled
54+
python export.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
55+
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
56+
cat ./output_aoti
57+
58+
echo "******************************************"
59+
echo "******* Emb: channel-wise quantized ******"
60+
echo "******************************************"
61+
python generate.py --dtype ${DTYPE} --quant '{"embedding" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
62+
cat ./output_eager
63+
python generate.py --dtype ${DTYPE} --compile --quant '{"embedding" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
64+
cat ./output_compiled
65+
python export.py --dtype ${DTYPE} --quant '{"embedding" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
66+
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
67+
cat ./output_aoti
68+
69+
echo "******************************************"
70+
echo "******** Emb: group-wise quantized *******"
71+
echo "******************************************"
72+
python generate.py --dtype ${DTYPE} --quant '{"embedding" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
73+
cat ./output_eager
74+
python generate.py --dtype ${DTYPE} --compile --quant '{"embedding" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
75+
cat ./output_compiled
76+
python export.py --dtype ${DTYPE} --quant '{"embedding" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
77+
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
78+
cat ./output_aoti
79+
80+
echo "******************************************"
81+
echo "******* INT8 channel-wise quantized ******"
82+
echo "******************************************"
83+
python generate.py --dtype ${DTYPE} --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
84+
cat ./output_eager
85+
python generate.py --dtype ${DTYPE} --compile --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
86+
cat ./output_compiled
87+
python export.py --dtype ${DTYPE} --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 0}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
88+
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
89+
cat ./output_aoti
90+
91+
echo "******************************************"
92+
echo "******** INT8 group-wise quantized *******"
93+
echo "******************************************"
94+
python generate.py --dtype ${DTYPE} --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
95+
cat ./output_eager
96+
python generate.py --dtype ${DTYPE} --compile --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
97+
cat ./output_compiled
98+
python export.py --dtype ${DTYPE} --quant '{"linear:int8" : {"bitwidth": 8, "group_size": 8}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
99+
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
100+
cat ./output_aoti
101+
102+
echo "******************************************"
103+
echo "******** INT4 group-wise quantized *******"
104+
echo "******************************************"
105+
if [ ${DTYPE} == float16 ]; then
106+
DTYPE=bfloat16
107+
fi
108+
109+
python generate.py --dtype ${DTYPE} --quant '{"linear:int4" : {"group_size": 32}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_eager
110+
cat ./output_eager
111+
python generate.py --dtype ${DTYPE} --compile --quant '{"linear:int4" : {"group_size": 32}}' --checkpoint-path ${MODEL_PATH} --temperature 0 > ./output_compiled
112+
cat ./output_compiled
113+
python export.py --dtype ${DTYPE} --quant '{"linear:int4" : {"group_size": 32}}' --checkpoint-path ${MODEL_PATH} --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so
114+
python generate.py --dtype ${DTYPE} --checkpoint-path ${MODEL_PATH} --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so > ./output_aoti
115+
cat ./output_aoti
116+
117+
echo "tests complete for ${DTYPE}"
118+
done
119+
120+
echo "tests complete for all dtypes!"

0 commit comments

Comments
 (0)