Skip to content

Commit 17b291a

Browse files
authored
convert-hf : Fix the encoding in the convert-hf-to-gguf-update.py (#8040)
1 parent abd894a commit 17b291a

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

convert-hf-to-gguf-update.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,15 +214,15 @@ def get_vocab_base_pre(self, tokenizer) -> str:
214214
"""
215215

216216
convert_py_pth = pathlib.Path("convert-hf-to-gguf.py")
217-
convert_py = convert_py_pth.read_text()
217+
convert_py = convert_py_pth.read_text(encoding="utf-8")
218218
convert_py = re.sub(
219219
r"(# Marker: Start get_vocab_base_pre)(.+?)( +# Marker: End get_vocab_base_pre)",
220220
lambda m: m.group(1) + src_func + m.group(3),
221221
convert_py,
222222
flags=re.DOTALL | re.MULTILINE,
223223
)
224224

225-
convert_py_pth.write_text(convert_py)
225+
convert_py_pth.write_text(convert_py, encoding="utf-8")
226226

227227
logger.info("+++ convert-hf-to-gguf.py was updated")
228228

0 commit comments

Comments
 (0)