@@ -67,13 +67,13 @@ class TOKENIZER_TYPE(IntEnum):
67
67
{"name" : "gpt-2" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/openai-community/gpt2" , },
68
68
{"name" : "phi" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/microsoft/phi-1" , },
69
69
{"name" : "stablelm" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b" , },
70
- {"name" : "qwen" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/Qwen/Qwen-7B" , },
71
70
{"name" : "mistral-bpe" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2" , },
72
71
{"name" : "mistral-spm" , "tokt" : TOKENIZER_TYPE .SPM , "repo" : "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2" , },
73
72
{"name" : "mixtral-bpe" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1" , },
74
73
{"name" : "mixtral-spm" , "tokt" : TOKENIZER_TYPE .SPM , "repo" : "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1" , },
75
74
{"name" : "refact" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/smallcloudai/Refact-1_6-base" , },
76
75
{"name" : "command-r" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/CohereForAI/c4ai-command-r-v01" , },
76
+ {"name" : "qwen" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/Qwen/Qwen-7B" , },
77
77
{"name" : "qwen2" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/Qwen/Qwen1.5-7B" , },
78
78
{"name" : "olmo" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/allenai/OLMo-1.7-7B-hf" , },
79
79
{"name" : "dbrx" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/databricks/dbrx-base" , },
0 commit comments