We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 6e6430c commit 345d78aCopy full SHA for 345d78a
convert.py
@@ -334,9 +334,9 @@ def load(model_plus: ModelPlus) -> Params:
334
class BpeVocab:
335
def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None:
336
self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read())
337
- try:
+ if isinstance(self.bpe_tokenizer.get('model'), dict):
338
self.vocab = self.bpe_tokenizer["model"]["vocab"]
339
- except KeyError:
+ else:
340
self.vocab = self.bpe_tokenizer
341
added_tokens: dict[str, int]
342
if fname_added_tokens is not None:
0 commit comments