Skip to content

Commit 4bce30c

Browse files
author
Joan Martinez
committed
fix: fix comments
1 parent 0fc775e commit 4bce30c

File tree

1 file changed

+8
-2
lines changed

1 file changed

+8
-2
lines changed

convert-hf-to-gguf.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:
420420
# NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script
421421
# or pull the latest version of the model from Huggingface
422422
# don't edit the hashes manually!
423+
if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
424+
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
425+
res = "llama-bpe"
423426
if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754":
424427
# ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base
425428
res = "deepseek-llm"
@@ -456,6 +459,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:
456459
if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166":
457460
# ref: https://huggingface.co/allenai/OLMo-1.7-7B-hf
458461
res = "olmo"
462+
if chkhsh == "a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79e":
463+
# ref: https://huggingface.co/databricks/dbrx-base
464+
res = "dbrx"
459465
if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f":
460466
# ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-en
461467
res = "jina-v2-en"
@@ -2451,10 +2457,10 @@ def get_tensors(self):
24512457
if 'gated_layer' in name:
24522458
d1 = data[:self.intermediate_size, :]
24532459
name1 = name.replace('gated_layers', 'gated_layers_w')
2454-
name1 = name.replace('up_gated_layer', 'gated_layers_v')
2460+
name1 = name1.replace('up_gated_layer', 'gated_layers_v')
24552461
d2 = data[self.intermediate_size:, :]
24562462
name2 = name.replace('gated_layers', 'gated_layers_v')
2457-
name2 = name.replace('up_gated_layer', 'gated_layers_w')
2463+
name2 = name2.replace('up_gated_layer', 'gated_layers_w')
24582464
yield name1, d1
24592465
yield name2, d2
24602466
continue

0 commit comments

Comments
 (0)