Skip to content

Commit 5c8155a

Browse files
ggerganovmglambda
ggerganov
authored and
mglambda
committed
llama : use LLAMA_TOKEN_NULL (ggml-org#11062)
ggml-ci
1 parent b38c89d commit 5c8155a

File tree

11 files changed

+55
-56
lines changed

11 files changed

+55
-56
lines changed

‎common/common.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -982,7 +982,7 @@ struct common_init_result common_init_from_params(common_params & params) {
982982
if (llama_model_has_encoder(model)) {
983983
llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size()));
984984
llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
985-
if (decoder_start_token_id == -1) {
985+
if (decoder_start_token_id == LLAMA_TOKEN_NULL) {
986986
decoder_start_token_id = bos;
987987
}
988988
tmp.clear();

‎common/ngram-cache.cpp

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -65,13 +65,13 @@ constexpr int draft_min_percent_strict[LLAMA_NGRAM_MAX] = {75, 66, 66, 66};
6565
static llama_token try_draft(common_ngram_cache & nc_static, const common_ngram ngram_static) {
6666
common_ngram_cache::iterator part_static_it = nc_static.find(ngram_static);
6767
if (part_static_it == nc_static.end()) {
68-
return -1;
68+
return LLAMA_TOKEN_NULL;
6969
}
7070
const common_ngram_cache_part part_static = part_static_it->second;
7171

7272
int max_count_static = 0;
7373
int sum_count_static = 0;
74-
llama_token max_token = -1;
74+
llama_token max_token = LLAMA_TOKEN_NULL;
7575

7676
for (std::pair<llama_token, int> token_count_static : part_static) {
7777
const llama_token token = token_count_static.first;
@@ -85,10 +85,10 @@ static llama_token try_draft(common_ngram_cache & nc_static, const common_ngram
8585
}
8686

8787
if (sum_count_static < draft_min_sample_size_lax[LLAMA_NGRAM_STATIC-1]) {
88-
return -1;
88+
return LLAMA_TOKEN_NULL;
8989
}
9090
if (100*max_count_static < draft_min_percent_lax[LLAMA_NGRAM_STATIC-1]*sum_count_static) {
91-
return -1;
91+
return LLAMA_TOKEN_NULL;
9292
}
9393
return max_token;
9494
}
@@ -98,9 +98,9 @@ static llama_token try_draft(
9898
common_ngram_cache & nc_primary, const std::vector<common_ngram> & ngrams_primary, common_ngram_cache_part & part_static,
9999
const int * min_sample_size, const int * min_percent) {
100100

101-
llama_token drafted_token = -1;
101+
llama_token drafted_token = LLAMA_TOKEN_NULL;
102102

103-
for (int i = ngrams_primary.size()-1; i >= 0 && drafted_token == -1; --i) {
103+
for (int i = ngrams_primary.size()-1; i >= 0 && drafted_token == LLAMA_TOKEN_NULL; --i) {
104104
const common_ngram ngram_primary = ngrams_primary[i];
105105

106106
common_ngram_cache::iterator part_primary_it = nc_primary.find(ngram_primary);
@@ -112,7 +112,7 @@ static llama_token try_draft(
112112
int max_count_primary = 0;
113113
int max_count_static = 0;
114114
int sum_count_primary = 0;
115-
llama_token max_token = -1;
115+
llama_token max_token = LLAMA_TOKEN_NULL;
116116

117117
for (std::pair<llama_token, int> token_count_primary : part_primary) {
118118
const llama_token token = token_count_primary.first;
@@ -154,7 +154,7 @@ void common_ngram_cache_draft(
154154
}
155155

156156
while ((int) draft.size()-1 < n_draft) {
157-
llama_token drafted_token = -1;
157+
llama_token drafted_token = LLAMA_TOKEN_NULL;
158158

159159
const int ngram_start_static = inp_size-LLAMA_NGRAM_STATIC + draft.size()-1;
160160
common_ngram ngram_static;
@@ -177,17 +177,17 @@ void common_ngram_cache_draft(
177177
}
178178
ngrams_cd.push_back(ngram_cd);
179179
}
180-
if (drafted_token == -1) {
180+
if (drafted_token == LLAMA_TOKEN_NULL) {
181181
drafted_token = try_draft(nc_context, ngrams_cd, part_static, draft_min_sample_size_lax, draft_min_percent_lax);
182182
}
183-
if (drafted_token == -1) {
183+
if (drafted_token == LLAMA_TOKEN_NULL) {
184184
drafted_token = try_draft(nc_dynamic, ngrams_cd, part_static, draft_min_sample_size_strict, draft_min_percent_strict);
185185
}
186-
if (drafted_token == -1) {
186+
if (drafted_token == LLAMA_TOKEN_NULL) {
187187
drafted_token = try_draft(nc_static, ngram_static);
188188
}
189189

190-
if (drafted_token == -1) {
190+
if (drafted_token == LLAMA_TOKEN_NULL) {
191191
break;
192192
}
193193

‎common/ngram-cache.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@ struct common_ngram {
1717

1818
common_ngram() {
1919
for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) {
20-
tokens[i] = -1;
20+
tokens[i] = LLAMA_TOKEN_NULL;
2121
}
2222
}
2323

2424
common_ngram(const llama_token * input, const int ngram_size) {
2525
for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) {
26-
tokens[i] = i < ngram_size ? input[i] : -1;
26+
tokens[i] = i < ngram_size ? input[i] : LLAMA_TOKEN_NULL;
2727
}
2828
}
2929

‎examples/batched/batched.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ int main(int argc, char ** argv) {
120120
}
121121

122122
llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
123-
if (decoder_start_token_id == -1) {
123+
if (decoder_start_token_id == LLAMA_TOKEN_NULL) {
124124
decoder_start_token_id = llama_token_bos(model);
125125
}
126126

‎examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -689,8 +689,8 @@ static void save_as_llama_model(
689689
gguf_set_val_u32(ctx, KV_TOKENIZER_UNK_ID, UNKNOWN_TOKEN_ID);
690690
gguf_set_val_u32(ctx, KV_TOKENIZER_BOS_ID, BOS_TOKEN_ID);
691691
gguf_set_val_u32(ctx, KV_TOKENIZER_EOS_ID, EOS_TOKEN_ID);
692-
gguf_set_val_u32(ctx, KV_TOKENIZER_SEP_ID, -1);
693-
gguf_set_val_u32(ctx, KV_TOKENIZER_PAD_ID, -1);
692+
gguf_set_val_u32(ctx, KV_TOKENIZER_SEP_ID, LLAMA_TOKEN_NULL);
693+
gguf_set_val_u32(ctx, KV_TOKENIZER_PAD_ID, LLAMA_TOKEN_NULL);
694694

695695
gguf_set_val_u32(ctx, KV_CONTEXT_LENGTH, model->hparams.n_ctx);
696696
gguf_set_val_u32(ctx, KV_EMBEDDING_LENGTH, model->hparams.n_embd);

‎examples/main/main.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,7 @@ int main(int argc, char ** argv) {
494494
}
495495

496496
llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
497-
if (decoder_start_token_id == -1) {
497+
if (decoder_start_token_id == LLAMA_TOKEN_NULL) {
498498
decoder_start_token_id = llama_token_bos(model);
499499
}
500500

@@ -831,7 +831,7 @@ int main(int argc, char ** argv) {
831831
// if user stop generation mid-way, we must add EOT to finish model's last response
832832
if (need_insert_eot && format_chat) {
833833
llama_token eot = llama_token_eot(model);
834-
embd_inp.push_back(eot == -1 ? llama_token_eos(model) : eot);
834+
embd_inp.push_back(eot == LLAMA_TOKEN_NULL ? llama_token_eos(model) : eot);
835835
need_insert_eot = false;
836836
}
837837

‎examples/server/utils.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ static std::string tokens_to_str(llama_context * ctx, Iter begin, Iter end) {
507507

508508
// format incomplete utf-8 multibyte character for output
509509
static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {
510-
std::string out = token == -1 ? "" : common_token_to_piece(ctx, token);
510+
std::string out = token == LLAMA_TOKEN_NULL ? "" : common_token_to_piece(ctx, token);
511511

512512
// if the size is 1 and first bit is 1, meaning it's a partial character
513513
// (size > 1 meaning it's already a known token)

‎include/llama.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434

3535
#define LLAMA_DEFAULT_SEED 0xFFFFFFFF
3636

37-
// TODO: use everywhere in the implementation
3837
#define LLAMA_TOKEN_NULL -1
3938

4039
#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'

‎src/llama-model.cpp

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1923,24 +1923,24 @@ void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {
19231923
LLAMA_LOG_INFO("%s: general.name = %s\n", __func__, model.name.c_str());
19241924

19251925
// special tokens
1926-
if (vocab.special_bos_id != -1) { LLAMA_LOG_INFO( "%s: BOS token = %d '%s'\n", __func__, vocab.special_bos_id, vocab.id_to_token[vocab.special_bos_id].text.c_str() ); }
1927-
if (vocab.special_eos_id != -1) { LLAMA_LOG_INFO( "%s: EOS token = %d '%s'\n", __func__, vocab.special_eos_id, vocab.id_to_token[vocab.special_eos_id].text.c_str() ); }
1928-
if (vocab.special_eot_id != -1) { LLAMA_LOG_INFO( "%s: EOT token = %d '%s'\n", __func__, vocab.special_eot_id, vocab.id_to_token[vocab.special_eot_id].text.c_str() ); }
1929-
if (vocab.special_eom_id != -1) { LLAMA_LOG_INFO( "%s: EOM token = %d '%s'\n", __func__, vocab.special_eom_id, vocab.id_to_token[vocab.special_eom_id].text.c_str() ); }
1930-
if (vocab.special_unk_id != -1) { LLAMA_LOG_INFO( "%s: UNK token = %d '%s'\n", __func__, vocab.special_unk_id, vocab.id_to_token[vocab.special_unk_id].text.c_str() ); }
1931-
if (vocab.special_sep_id != -1) { LLAMA_LOG_INFO( "%s: SEP token = %d '%s'\n", __func__, vocab.special_sep_id, vocab.id_to_token[vocab.special_sep_id].text.c_str() ); }
1932-
if (vocab.special_pad_id != -1) { LLAMA_LOG_INFO( "%s: PAD token = %d '%s'\n", __func__, vocab.special_pad_id, vocab.id_to_token[vocab.special_pad_id].text.c_str() ); }
1933-
if (vocab.special_cls_id != -1) { LLAMA_LOG_INFO( "%s: CLS token = %d '%s'\n", __func__, vocab.special_cls_id, vocab.id_to_token[vocab.special_cls_id].text.c_str() ); }
1934-
if (vocab.special_mask_id != -1) { LLAMA_LOG_INFO( "%s: MASK token = %d '%s'\n", __func__, vocab.special_mask_id, vocab.id_to_token[vocab.special_mask_id].text.c_str() ); }
1935-
1936-
if (vocab.linefeed_id != -1) { LLAMA_LOG_INFO( "%s: LF token = %d '%s'\n", __func__, vocab.linefeed_id, vocab.id_to_token[vocab.linefeed_id].text.c_str() ); }
1937-
1938-
if (vocab.special_fim_pre_id != -1) { LLAMA_LOG_INFO( "%s: FIM PRE token = %d '%s'\n", __func__, vocab.special_fim_pre_id, vocab.id_to_token[vocab.special_fim_pre_id].text.c_str() ); }
1939-
if (vocab.special_fim_suf_id != -1) { LLAMA_LOG_INFO( "%s: FIM SUF token = %d '%s'\n", __func__, vocab.special_fim_suf_id, vocab.id_to_token[vocab.special_fim_suf_id].text.c_str() ); }
1940-
if (vocab.special_fim_mid_id != -1) { LLAMA_LOG_INFO( "%s: FIM MID token = %d '%s'\n", __func__, vocab.special_fim_mid_id, vocab.id_to_token[vocab.special_fim_mid_id].text.c_str() ); }
1941-
if (vocab.special_fim_pad_id != -1) { LLAMA_LOG_INFO( "%s: FIM PAD token = %d '%s'\n", __func__, vocab.special_fim_pad_id, vocab.id_to_token[vocab.special_fim_pad_id].text.c_str() ); }
1942-
if (vocab.special_fim_rep_id != -1) { LLAMA_LOG_INFO( "%s: FIM REP token = %d '%s'\n", __func__, vocab.special_fim_rep_id, vocab.id_to_token[vocab.special_fim_rep_id].text.c_str() ); }
1943-
if (vocab.special_fim_sep_id != -1) { LLAMA_LOG_INFO( "%s: FIM SEP token = %d '%s'\n", __func__, vocab.special_fim_sep_id, vocab.id_to_token[vocab.special_fim_sep_id].text.c_str() ); }
1926+
if (vocab.special_bos_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: BOS token = %d '%s'\n", __func__, vocab.special_bos_id, vocab.id_to_token[vocab.special_bos_id].text.c_str() ); }
1927+
if (vocab.special_eos_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: EOS token = %d '%s'\n", __func__, vocab.special_eos_id, vocab.id_to_token[vocab.special_eos_id].text.c_str() ); }
1928+
if (vocab.special_eot_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: EOT token = %d '%s'\n", __func__, vocab.special_eot_id, vocab.id_to_token[vocab.special_eot_id].text.c_str() ); }
1929+
if (vocab.special_eom_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: EOM token = %d '%s'\n", __func__, vocab.special_eom_id, vocab.id_to_token[vocab.special_eom_id].text.c_str() ); }
1930+
if (vocab.special_unk_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: UNK token = %d '%s'\n", __func__, vocab.special_unk_id, vocab.id_to_token[vocab.special_unk_id].text.c_str() ); }
1931+
if (vocab.special_sep_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: SEP token = %d '%s'\n", __func__, vocab.special_sep_id, vocab.id_to_token[vocab.special_sep_id].text.c_str() ); }
1932+
if (vocab.special_pad_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: PAD token = %d '%s'\n", __func__, vocab.special_pad_id, vocab.id_to_token[vocab.special_pad_id].text.c_str() ); }
1933+
if (vocab.special_cls_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: CLS token = %d '%s'\n", __func__, vocab.special_cls_id, vocab.id_to_token[vocab.special_cls_id].text.c_str() ); }
1934+
if (vocab.special_mask_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: MASK token = %d '%s'\n", __func__, vocab.special_mask_id, vocab.id_to_token[vocab.special_mask_id].text.c_str() ); }
1935+
1936+
if (vocab.linefeed_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: LF token = %d '%s'\n", __func__, vocab.linefeed_id, vocab.id_to_token[vocab.linefeed_id].text.c_str() ); }
1937+
1938+
if (vocab.special_fim_pre_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: FIM PRE token = %d '%s'\n", __func__, vocab.special_fim_pre_id, vocab.id_to_token[vocab.special_fim_pre_id].text.c_str() ); }
1939+
if (vocab.special_fim_suf_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: FIM SUF token = %d '%s'\n", __func__, vocab.special_fim_suf_id, vocab.id_to_token[vocab.special_fim_suf_id].text.c_str() ); }
1940+
if (vocab.special_fim_mid_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: FIM MID token = %d '%s'\n", __func__, vocab.special_fim_mid_id, vocab.id_to_token[vocab.special_fim_mid_id].text.c_str() ); }
1941+
if (vocab.special_fim_pad_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: FIM PAD token = %d '%s'\n", __func__, vocab.special_fim_pad_id, vocab.id_to_token[vocab.special_fim_pad_id].text.c_str() ); }
1942+
if (vocab.special_fim_rep_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: FIM REP token = %d '%s'\n", __func__, vocab.special_fim_rep_id, vocab.id_to_token[vocab.special_fim_rep_id].text.c_str() ); }
1943+
if (vocab.special_fim_sep_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( "%s: FIM SEP token = %d '%s'\n", __func__, vocab.special_fim_sep_id, vocab.id_to_token[vocab.special_fim_sep_id].text.c_str() ); }
19441944

19451945
for (const auto & id : vocab.special_eog_ids) {
19461946
LLAMA_LOG_INFO( "%s: EOG token = %d '%s'\n", __func__, id, vocab.id_to_token[id].text.c_str() );

‎src/llama-sampling.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ static void llama_sampler_top_k_impl(llama_token_data_array * cur_p, int32_t k)
257257
for (int i = 0; i < (int)cur_p->size; ++i) {
258258
const float val = cur_p->data[i].logit;
259259
int ib = int(bucket_scale * val + bucket_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
260-
ib = std::max(0, std::min(nbuckets-1, ib));
260+
ib = std::max(0, std::min(nbuckets - 1, ib));
261261
bucket_idx[i] = ib;
262262
++histo[ib];
263263
}
@@ -280,13 +280,13 @@ static void llama_sampler_top_k_impl(llama_token_data_array * cur_p, int32_t k)
280280
for (int i = 0; i < (int)cur_p->size; ++i) {
281281
int j = bucket_idx[i];
282282
if (j >= ib) {
283-
*bucket_ptrs[nbuckets-1-j]++ = cur_p->data[i];
283+
*bucket_ptrs[nbuckets - 1 - j]++ = cur_p->data[i];
284284
}
285285
}
286286

287287
ptr = tmp_tokens.data();
288288
int ndone = 0;
289-
for (int j = nbuckets-1; j > ib; --j) {
289+
for (int j = nbuckets - 1; j > ib; --j) {
290290
std::sort(ptr, ptr + histo[j], comp);
291291
ptr += histo[j];
292292
ndone += histo[j];
@@ -1832,7 +1832,7 @@ static void llama_sampler_dry_apply(struct llama_sampler * smpl, llama_token_dat
18321832
ctx->dry_repeat_count[last - k] = std::min(n, rep_limit);
18331833
if (n > 0) {
18341834
lt = k;
1835-
rt = k+n-1;
1835+
rt = k + n - 1;
18361836
}
18371837
} else {
18381838
// If k is inside the current Z-box, consider two cases.

‎src/llama-vocab.cpp

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -497,7 +497,7 @@ struct llm_tokenizer_bpe_session {
497497

498498
bool append_bos(std::vector<llama_vocab::id> & output) const {
499499
if (vocab.tokenizer_add_bos) {
500-
GGML_ASSERT(vocab.special_bos_id != -1);
500+
GGML_ASSERT(vocab.special_bos_id != LLAMA_TOKEN_NULL);
501501
output.push_back(vocab.special_bos_id);
502502
return true;
503503
}
@@ -506,7 +506,7 @@ struct llm_tokenizer_bpe_session {
506506

507507
bool append_eos(std::vector<llama_vocab::id> & output) const {
508508
if (vocab.tokenizer_add_eos) {
509-
GGML_ASSERT(vocab.special_eos_id != -1);
509+
GGML_ASSERT(vocab.special_eos_id != LLAMA_TOKEN_NULL);
510510
output.push_back(vocab.special_eos_id);
511511
return true;
512512
}
@@ -1403,7 +1403,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
14031403
if (source == 0) {
14041404
buffer.erase_after(buffer.before_begin());
14051405
} else {
1406-
buffer.erase_after(std::next(buffer.begin(), (source-1)));
1406+
buffer.erase_after(std::next(buffer.begin(), (source - 1)));
14071407
}
14081408

14091409
// repeat for the right side
@@ -1417,7 +1417,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
14171417
if (source == 0) {
14181418
buffer.erase_after(buffer.before_begin());
14191419
} else {
1420-
buffer.erase_after(std::next(buffer.begin(), (source-1)));
1420+
buffer.erase_after(std::next(buffer.begin(), (source - 1)));
14211421
}
14221422
break;
14231423
}
@@ -1454,7 +1454,7 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
14541454
bool is_prev_special = true; // prefix with space if first token
14551455

14561456
if (add_special && vocab.tokenizer_add_bos) {
1457-
GGML_ASSERT(vocab.special_bos_id != -1);
1457+
GGML_ASSERT(vocab.special_bos_id != LLAMA_TOKEN_NULL);
14581458
output.push_back(vocab.special_bos_id);
14591459
is_prev_special = true;
14601460
}
@@ -1489,7 +1489,7 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
14891489
}
14901490

14911491
if (add_special && vocab.tokenizer_add_eos) {
1492-
GGML_ASSERT(vocab.special_eos_id != -1);
1492+
GGML_ASSERT(vocab.special_eos_id != LLAMA_TOKEN_NULL);
14931493
output.push_back(vocab.special_eos_id);
14941494
}
14951495
} break;
@@ -1522,7 +1522,7 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
15221522
case LLAMA_VOCAB_TYPE_WPM:
15231523
{
15241524
if (add_special) {
1525-
GGML_ASSERT(vocab.special_cls_id != -1);
1525+
GGML_ASSERT(vocab.special_cls_id != LLAMA_TOKEN_NULL);
15261526
output.push_back(vocab.special_cls_id);
15271527
}
15281528

@@ -1542,14 +1542,14 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
15421542
}
15431543

15441544
if (add_special) {
1545-
GGML_ASSERT(vocab.special_sep_id != -1);
1545+
GGML_ASSERT(vocab.special_sep_id != LLAMA_TOKEN_NULL);
15461546
output.push_back(vocab.special_sep_id);
15471547
}
15481548
} break;
15491549
case LLAMA_VOCAB_TYPE_UGM:
15501550
{
15511551
if (add_special && vocab.tokenizer_add_bos) {
1552-
GGML_ASSERT(vocab.special_bos_id != -1);
1552+
GGML_ASSERT(vocab.special_bos_id != LLAMA_TOKEN_NULL);
15531553
output.push_back(vocab.special_bos_id);
15541554
}
15551555
llm_tokenizer_ugm_session session(vocab);
@@ -1574,7 +1574,7 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
15741574
}
15751575

15761576
if (add_special && vocab.tokenizer_add_eos) {
1577-
GGML_ASSERT(vocab.special_eos_id != -1);
1577+
GGML_ASSERT(vocab.special_eos_id != LLAMA_TOKEN_NULL);
15781578
output.push_back(vocab.special_eos_id);
15791579
}
15801580
} break;
@@ -1642,7 +1642,7 @@ llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, lla
16421642
}
16431643

16441644
bool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token) {
1645-
return token != -1 && vocab.special_eog_ids.count(token) > 0;
1645+
return token != LLAMA_TOKEN_NULL && vocab.special_eog_ids.count(token) > 0;
16461646
}
16471647

16481648
bool llama_token_is_control_impl(const struct llama_vocab & vocab, llama_token token) {
@@ -1881,7 +1881,7 @@ int32_t llama_detokenize_impl(
18811881
}
18821882

18831883
if (remove_special && vocab.tokenizer_add_eos) {
1884-
if (n_tokens > 0 && tokens[n_tokens-1] == vocab.special_eos_id) {
1884+
if (n_tokens > 0 && tokens[n_tokens - 1] == vocab.special_eos_id) {
18851885
n_tokens--;
18861886
}
18871887
}

0 commit comments

Comments
 (0)