starcoder : fix pre-tokenizer

ggerganov · ggerganov · commit cf00fe1ea325 · 2024-05-02T11:00:15.000+03:00
diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py
@@ -189,6 +189,7 @@ def download_file_with_auth(url, token, save_path):
 # generate tests for each tokenizer model
 
 tests = [
+    "ied 4 ½ months"
     "",
     " ",
     "  ",
diff --git a/llama.cpp b/llama.cpp
@@ -12235,6 +12235,11 @@ struct llm_tokenizer_bpe {
                         });
                         break;
                     case LLAMA_VOCAB_PRE_TYPE_STARCODER:
+                        word_collection = unicode_regex_split(text, {
+                            "\\p{N}",
+                            "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+                        });
+                        break;
                     case LLAMA_VOCAB_PRE_TYPE_GPT2:
                         word_collection = unicode_regex_split(text, {
                             "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
diff --git a/models/ggml-vocab-bert-bge.gguf.inp b/models/ggml-vocab-bert-bge.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-bert-bge.gguf.out b/models/ggml-vocab-bert-bge.gguf.out
@@ -1,4 +1,4 @@
-
+ 29464 2094 1018 1092 2706
 
 
 
diff --git a/models/ggml-vocab-deepseek-coder.gguf.inp b/models/ggml-vocab-deepseek-coder.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-deepseek-coder.gguf.out b/models/ggml-vocab-deepseek-coder.gguf.out
@@ -1,4 +1,4 @@
-
+ 1050 207 19 207 19192 4217
  207
  243
  315
diff --git a/models/ggml-vocab-deepseek-llm.gguf.inp b/models/ggml-vocab-deepseek-llm.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-deepseek-llm.gguf.out b/models/ggml-vocab-deepseek-llm.gguf.out
@@ -1,4 +1,4 @@
-
+ 1052 207 19 207 19109 4223
  207
  243
  300
diff --git a/models/ggml-vocab-falcon.gguf.inp b/models/ggml-vocab-falcon.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-falcon.gguf.out b/models/ggml-vocab-falcon.gguf.out
@@ -1,4 +1,4 @@
-
+ 878 204 31 3068 133 2137
  204
  258
  466
diff --git a/models/ggml-vocab-gpt-2.gguf.inp b/models/ggml-vocab-gpt-2.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-gpt-2.gguf.out b/models/ggml-vocab-gpt-2.gguf.out
@@ -1,4 +1,4 @@
-
+ 798 604 25208 1933
  220
  220 220
  220 220 220
diff --git a/models/ggml-vocab-llama-bpe.gguf.inp b/models/ggml-vocab-llama-bpe.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-llama-bpe.gguf.out b/models/ggml-vocab-llama-bpe.gguf.out
@@ -1,4 +1,4 @@
-
+ 1142 220 19 220 27154 4038
  220
  256
  262
diff --git a/models/ggml-vocab-llama-spm.gguf.inp b/models/ggml-vocab-llama-spm.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-llama-spm.gguf.out b/models/ggml-vocab-llama-spm.gguf.out
@@ -1,4 +1,4 @@
-
+ 474 287 29871 29946 29871 30226 7378
  259
  1678
  268
diff --git a/models/ggml-vocab-mpt.gguf.inp b/models/ggml-vocab-mpt.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-mpt.gguf.out b/models/ggml-vocab-mpt.gguf.out
@@ -1,4 +1,4 @@
-
+ 728 577 24142 2607
  209
  50276
  50275
diff --git a/models/ggml-vocab-phi-3.gguf.inp b/models/ggml-vocab-phi-3.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-phi-3.gguf.out b/models/ggml-vocab-phi-3.gguf.out
@@ -1,4 +1,4 @@
-
+ 474 287 29871 29946 29871 30226 7378
  259
  1678
  268
diff --git a/models/ggml-vocab-starcoder.gguf.inp b/models/ggml-vocab-starcoder.gguf.inp
@@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
  
 __ggml_vocab_test__
diff --git a/models/ggml-vocab-starcoder.gguf.out b/models/ggml-vocab-starcoder.gguf.out
@@ -1,4 +1,4 @@
-
+ 4850 244 57 244 162 159 17722
  244
  280
  283

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ied 4 ½ months`
`2`	`2`	`__ggml_vocab_test__`
`3`	`3`
`4`	`4`	`__ggml_vocab_test__`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 29464 2094 1018 1092 2706`
`2`	`2`
`3`	`3`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 1050 207 19 207 19192 4217`
`2`	`2`	`207`
`3`	`3`	`243`
`4`	`4`	`315`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 1052 207 19 207 19109 4223`
`2`	`2`	`207`
`3`	`3`	`243`
`4`	`4`	`300`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 878 204 31 3068 133 2137`
`2`	`2`	`204`
`3`	`3`	`258`
`4`	`4`	`466`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 798 604 25208 1933`
`2`	`2`	`220`
`3`	`3`	`220 220`
`4`	`4`	`220 220 220`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 1142 220 19 220 27154 4038`
`2`	`2`	`220`
`3`	`3`	`256`
`4`	`4`	`262`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 474 287 29871 29946 29871 30226 7378`
`2`	`2`	`259`
`3`	`3`	`1678`
`4`	`4`	`268`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 728 577 24142 2607`
`2`	`2`	`209`
`3`	`3`	`50276`
`4`	`4`	`50275`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-`
	`1`	`+ 4850 244 57 244 162 159 17722`
`2`	`2`	`244`
`3`	`3`	`280`
`4`	`4`	`283`