Skip to content

Commit d6fe269

Browse files
committed
llama : fix command-r detokenization
1 parent 31a1b0e commit d6fe269

File tree

1 file changed

+1
-0
lines changed

1 file changed

+1
-0
lines changed

src/llama.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5407,6 +5407,7 @@ static void llm_load_vocab(
54075407
} else if (
54085408
tokenizer_pre == "command-r") {
54095409
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;
5410+
vocab.tokenizer_clean_spaces = false;
54105411
} else if (
54115412
tokenizer_pre == "qwen2") {
54125413
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2;

0 commit comments

Comments
 (0)