support Llama-3_1-Nemotron-51B

Author: ymcki

gguf-py/gguf/vocab.py

@@ -109,9 +109,7 @@ def _set_special_token(self, typ: str, tid: Any) -> None:
         if tid < 0:
             raise ValueError(f'invalid value for special token type {typ}: {tid}')
         if self.n_vocab is None or tid < self.n_vocab:
-            if typ in self.special_token_ids:
-                return
-            self.special_token_ids[typ] = tid
+            self.special_token_ids[typ] = tid # allow override
             return
         logger.warning(f'Special token type {typ}, id {tid} out of range, must be under {self.n_vocab} - skipping')
 
@@ -188,7 +186,14 @@ def _try_load_from_config_json(self, path: Path) -> bool:
         with open(config_file, encoding = 'utf-8') as f:
             config = json.load(f)
         for typ in self.special_token_types:
-            self._set_special_token(typ, config.get(f'{typ}_token_id'))
+            # load eos tokens when it is an array
+            if typ == 'eos' and isinstance(config.get(f'{typ}_token_id'), list):
+                eos_ids = config.get(f'{typ}_token_id')
+                self._set_special_token('eos', eos_ids[0])
+                self._set_special_token('eom', eos_ids[1])
+                self._set_special_token('eot', eos_ids[2])
+            else:
+                self._set_special_token(typ, config.get(f'{typ}_token_id'))
         return True
 
 

src/llama.cpp

@@ -7680,7 +7680,6 @@ static bool llm_load_tensors(
                     }
 
                     for (int i = 0; i < n_layer; ++i) {
-                        auto & layer = model.layers[i];
                         auto & layer = model.layers[i];
                         const int64_t n_embd_k_gqa  = hparams.n_embd_k_gqa(i);
                         const int64_t n_embd_v_gqa  = hparams.n_embd_v_gqa(i);
@@ -10806,6 +10805,8 @@ struct llm_build_context {
         const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
         for (int il = 0; il < n_layer; ++il) {
             struct ggml_tensor * inpSA = inpL;
+            const int64_t n_head_kv = hparams.n_head_kv(il);
+            const int64_t n_head    = hparams.n_head(il);
 
             if (n_head == 0) // attention-free layer of Llama-3_1-Nemotron-51B
                 cur = inpL;