Skip to content

Commit 1e09d92

Browse files
committed
correct the llama_tokenize param0 from self.model to self.vocab
1 parent 6eb78ec commit 1e09d92

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

llama_cpp/_internals.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,13 +168,13 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool):
168168
n_ctx = self.n_ctx_train()
169169
tokens = (llama_cpp.llama_token * n_ctx)()
170170
n_tokens = llama_cpp.llama_tokenize(
171-
self.model, text, len(text), tokens, n_ctx, add_bos, special
171+
self.vocab, text, len(text), tokens, n_ctx, add_bos, special
172172
)
173173
if n_tokens < 0:
174174
n_tokens = abs(n_tokens)
175175
tokens = (llama_cpp.llama_token * n_tokens)()
176176
n_tokens = llama_cpp.llama_tokenize(
177-
self.model, text, len(text), tokens, n_tokens, add_bos, special
177+
self.vocab, text, len(text), tokens, n_tokens, add_bos, special
178178
)
179179
if n_tokens < 0:
180180
raise RuntimeError(

0 commit comments

Comments
 (0)