@@ -420,6 +420,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:
420
420
# NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script
421
421
# or pull the latest version of the model from Huggingface
422
422
# don't edit the hashes manually!
423
+ if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5" :
424
+ # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
425
+ res = "llama-bpe"
423
426
if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754" :
424
427
# ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base
425
428
res = "deepseek-llm"
@@ -456,6 +459,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:
456
459
if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166" :
457
460
# ref: https://huggingface.co/allenai/OLMo-1.7-7B-hf
458
461
res = "olmo"
462
+ if chkhsh == "a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79e" :
463
+ # ref: https://huggingface.co/databricks/dbrx-base
464
+ res = "dbrx"
459
465
if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f" :
460
466
# ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-en
461
467
res = "jina-v2-en"
@@ -2451,10 +2457,10 @@ def get_tensors(self):
2451
2457
if 'gated_layer' in name :
2452
2458
d1 = data [:self .intermediate_size , :]
2453
2459
name1 = name .replace ('gated_layers' , 'gated_layers_w' )
2454
- name1 = name .replace ('up_gated_layer' , 'gated_layers_v' )
2460
+ name1 = name1 .replace ('up_gated_layer' , 'gated_layers_v' )
2455
2461
d2 = data [self .intermediate_size :, :]
2456
2462
name2 = name .replace ('gated_layers' , 'gated_layers_v' )
2457
- name2 = name .replace ('up_gated_layer' , 'gated_layers_w' )
2463
+ name2 = name2 .replace ('up_gated_layer' , 'gated_layers_w' )
2458
2464
yield name1 , d1
2459
2465
yield name2 , d2
2460
2466
continue
0 commit comments