add OLMo to pre-tokenizer generation script

nopperl · nopperl · commit dba3d2430ffc · 2024-05-05T21:03:34.000+02:00
diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py
@@ -67,6 +67,7 @@ class TOKENIZER_TYPE(IntEnum):
     {"name": "gpt-2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
     {"name": "refact",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", },
     {"name": "command-r",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },
+    {"name": "olmo",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },
 ]
 
 # make directory "models/tokenizers" if it doesn't exist

Original file line number	Diff line number	Diff line change
`@@ -67,6 +67,7 @@ class TOKENIZER_TYPE(IntEnum):`
`67`	`67`	`{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },`
`68`	`68`	`{"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", },`
`69`	`69`	`{"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },`
	`70`	`+ {"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },`
`70`	`71`	`]`
`71`	`72`
`72`	`73`	`# make directory "models/tokenizers" if it doesn't exist`