quantization

Author: lucylq

README.md

@@ -233,6 +233,7 @@ python3 torchchat.py export stories15M --output-pte-path stories15M.pte
 # Execute
 python3 torchchat.py generate --device cpu --pte-path stories15M.pte --prompt "Hello my name is"
 ```
+* Note, to export a llama model, it's recommended to quantize with `--quantize '{"embedding": {"bitwidth": 4, "groupsize":32}, “linear:a8w4dq”: {“groupsize” : 256}}'`
 
 See below under [Mobile Execution](#run-mobile) if you want to deploy and execute a model in your iOS or Android app.
 

docs/quantization.md

@@ -214,12 +214,12 @@ python3 generate.py [--compile] --checkpoint-path ${MODEL_PATH} --prompt "Hello,
 ```
 
 ```
-python3 export.py --checkpoint-path ${MODEL_PATH} -d fp32 --quantize '{"linear:int4": {"groupsize" : 32} }' [ --output-pte-path ${MODEL_OUT}/${MODEL_NAME}_int4-gw32.pte | --output-dso-path ${MODEL_OUT}/${MODEL_NAME}_int4-gw32.dso]
+python3 export.py --checkpoint-path ${MODEL_PATH} -d fp32 --quantize '{"linear:int4": {"groupsize" : 32} }' --output-dso-path ${MODEL_OUT}/${MODEL_NAME}_int4-gw32.dso
 ```
 Now you can run your model with the same command as before:
 
 ```
-python3 generate.py [ --pte-path ${MODEL_OUT}/${MODEL_NAME}_int4-gw32.pte | --dso-path ${MODEL_OUT}/${MODEL_NAME}_int4-gw32.dso]  --prompt "Hello my name is"
+python3 generate.py --dso-path ${MODEL_OUT}/${MODEL_NAME}_int4-gw32.dso  --prompt "Hello my name is"
 ```
 
 ## 4-Bit Integer Linear Quantization  (a8w4dq)
@@ -247,12 +247,12 @@ python3 generate.py [--compile] --checkpoint-path ${MODEL_PATH} --prompt "Hello,
 ```
 
 ```
-python3 export.py --checkpoint-path ${MODEL_PATH} -d fp32 --quantize '{"linear:gptq": {"groupsize" : 32} }' [ --output-pte-path ${MODEL_OUT}/${MODEL_NAME}_gptq.pte | ...dso... ]
+python3 export.py --checkpoint-path ${MODEL_PATH} -d fp32 --quantize '{"linear:gptq": {"groupsize" : 32} }' --output-dso-path ${MODEL_OUT}/${MODEL_NAME}_gptq.dso
 ```
 Now you can run your model with the same command as before:
 
 ```
-python3 generate.py [ --pte-path ${MODEL_OUT}/${MODEL_NAME}_gptq.pte | ...dso...]  --prompt "Hello my name is"
+python3 generate.py --dso-path ${MODEL_OUT}/${MODEL_NAME}_gptq.dso  --prompt "Hello my name is"
 ```
 
 ## 4-bit Integer Linear Quantization with HQQ (hqq)
@@ -267,12 +267,12 @@ python3 generate.py [--compile] --checkpoint-path ${MODEL_PATH} --prompt "Hello,
 ```
 
 ```
-python3 export.py --checkpoint-path ${MODEL_PATH} -d fp32 --quantize '{"linear:hqq": {"groupsize" : 32} }' [ --output-pte-path ${MODEL_OUT}/${MODEL_NAME}_hqq.pte | ...dso... ]
+python3 export.py --checkpoint-path ${MODEL_PATH} -d fp32 --quantize '{"linear:hqq": {"groupsize" : 32} }' --output-dso-path ${MODEL_OUT}/${MODEL_NAME}_hqq.dso
 ```
 Now you can run your model with the same command as before:
 
 ```
-python3 generate.py [ --pte-path ${MODEL_OUT}/${MODEL_NAME}_hqq.pte | ...dso...]  --prompt "Hello my name is"
+python3 generate.py --dso-path ${MODEL_OUT}/${MODEL_NAME}_hqq.dso --prompt "Hello my name is"
 
 
 ## Adding additional quantization schemes

download.py

@@ -93,7 +93,6 @@ def download_and_convert(
         # overwriting if necessary.
         if os.path.isdir(model_dir):
             shutil.rmtree(model_dir)
-        os.makedirs(model_dir, exist_ok=True)
         shutil.move(temp_dir, model_dir)
 
     finally: