Skip to content

Commit f68664a

Browse files
author
Sang-Kil Park
authored
convert : fix TypeError on GPT-2 vocab.json (#5288)
1 parent 213d143 commit f68664a

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

convert.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -334,9 +334,9 @@ def load(model_plus: ModelPlus) -> Params:
334334
class BpeVocab:
335335
def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None:
336336
self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read())
337-
try:
337+
if isinstance(self.bpe_tokenizer.get('model'), dict):
338338
self.vocab = self.bpe_tokenizer["model"]["vocab"]
339-
except KeyError:
339+
else:
340340
self.vocab = self.bpe_tokenizer
341341
added_tokens: dict[str, int]
342342
if fname_added_tokens is not None:

0 commit comments

Comments
 (0)