Skip to content

Commit c90dbe0

Browse files
authored
Fix per token atrributes bits (#7749)
1 parent b90dc56 commit c90dbe0

File tree

1 file changed

+10
-10
lines changed

1 file changed

+10
-10
lines changed

llama.h

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,16 +109,16 @@ extern "C" {
109109

110110
enum llama_token_attr {
111111
LLAMA_TOKEN_ATTR_UNDEFINED = 0,
112-
LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 1,
113-
LLAMA_TOKEN_ATTR_UNUSED = 1 << 2,
114-
LLAMA_TOKEN_ATTR_NORMAL = 1 << 3,
115-
LLAMA_TOKEN_ATTR_CONTROL = 1 << 4, // SPECIAL?
116-
LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 5,
117-
LLAMA_TOKEN_ATTR_BYTE = 1 << 6,
118-
LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 7,
119-
LLAMA_TOKEN_ATTR_LSTRIP = 1 << 8,
120-
LLAMA_TOKEN_ATTR_RSTRIP = 1 << 9,
121-
LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 10,
112+
LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,
113+
LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,
114+
LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,
115+
LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?
116+
LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
117+
LLAMA_TOKEN_ATTR_BYTE = 1 << 5,
118+
LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,
119+
LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,
120+
LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,
121+
LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,
122122
};
123123

124124
// model file types

0 commit comments

Comments
 (0)