Skip to content

Commit b8c80df

Browse files
KerfuffleV2KerfuffleV2
committed
gguf-py: Refactor and add file reading support
1 parent 0a7c980 commit b8c80df

17 files changed

+1502
-1155
lines changed

‎convert-baichuan-hf-to-gguf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from sentencepiece import SentencePieceProcessor # type: ignore[import]
1717

1818
if 'NO_LOCAL_GGUF' not in os.environ:
19-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
19+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
2020
import gguf
2121

2222

‎convert-bloom-hf-to-gguf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
from transformers import AutoTokenizer # type: ignore[import]
1818

1919
if 'NO_LOCAL_GGUF' not in os.environ:
20-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
20+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
2121
import gguf
2222

2323

‎convert-falcon-hf-to-gguf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
from transformers import AutoTokenizer # type: ignore[import]
1818

1919
if 'NO_LOCAL_GGUF' not in os.environ:
20-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
20+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
2121
import gguf
2222

2323

‎convert-gptneox-hf-to-gguf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from transformers import AutoTokenizer # type: ignore[import]
1717

1818
if 'NO_LOCAL_GGUF' not in os.environ:
19-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
19+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
2020
import gguf
2121

2222

‎convert-llama-ggml-to-gguf.py

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -12,29 +12,9 @@
1212

1313
import os
1414
if 'NO_LOCAL_GGUF' not in os.environ:
15-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
15+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
1616
import gguf
1717

18-
# Note: Does not support GGML_QKK_64
19-
QK_K = 256
20-
# Items here are (block size, type size)
21-
GGML_QUANT_SIZES = {
22-
gguf.GGMLQuantizationType.F32 : (1, 4),
23-
gguf.GGMLQuantizationType.F16 : (1, 2),
24-
gguf.GGMLQuantizationType.Q4_0 : (32, 2 + 16),
25-
gguf.GGMLQuantizationType.Q4_1 : (32, 2 + 2 + 16),
26-
gguf.GGMLQuantizationType.Q5_0 : (32, 2 + 4 + 16),
27-
gguf.GGMLQuantizationType.Q5_1 : (32, 2 + 2 + 4 + 16),
28-
gguf.GGMLQuantizationType.Q8_0 : (32, 2 + 32),
29-
gguf.GGMLQuantizationType.Q8_1 : (32, 4 + 4 + 32),
30-
gguf.GGMLQuantizationType.Q2_K : (256, 2 + 2 + QK_K // 16 + QK_K // 4),
31-
gguf.GGMLQuantizationType.Q3_K : (256, 2 + QK_K // 4 + QK_K // 8 + 12),
32-
gguf.GGMLQuantizationType.Q4_K : (256, 2 + 2 + QK_K // 2 + 12),
33-
gguf.GGMLQuantizationType.Q5_K : (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),
34-
gguf.GGMLQuantizationType.Q6_K : (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),
35-
gguf.GGMLQuantizationType.Q8_K : (256, 4 + QK_K + QK_K // 8),
36-
}
37-
3818
class GGMLFormat(IntEnum):
3919
GGML = 0
4020
GGMF = 1
@@ -125,7 +105,7 @@ def load(self, data, offset):
125105
(n_dims, name_len, dtype) = struct.unpack('<3I', data[offset:offset + 12])
126106
assert n_dims >= 0 and n_dims <= 4, f'Invalid tensor dimensions {n_dims}'
127107
assert name_len < 4096, 'Absurd tensor name length'
128-
quant = GGML_QUANT_SIZES.get(dtype)
108+
quant = gguf.GGML_QUANT_SIZES.get(dtype)
129109
assert quant is not None, 'Unknown tensor type'
130110
(blksize, tysize) = quant
131111
offset += 12

‎convert-mpt-hf-to-gguf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from transformers import AutoTokenizer # type: ignore[import]
1717

1818
if 'NO_LOCAL_GGUF' not in os.environ:
19-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
19+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
2020
import gguf
2121

2222

‎convert-persimmon-to-gguf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from pathlib import Path
77
from sentencepiece import SentencePieceProcessor
88
if 'NO_LOCAL_GGUF' not in os.environ:
9-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
9+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
1010
import gguf
1111

1212
def _flatten_dict(dct, tensors, prefix=None):

‎convert-starcoder-hf-to-gguf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from transformers import AutoTokenizer # type: ignore[import]
1717

1818
if 'NO_LOCAL_GGUF' not in os.environ:
19-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
19+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
2020
import gguf
2121

2222

‎convert.py

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,9 @@
33

44
import argparse
55
import concurrent.futures
6-
import copy
76
import enum
87
import faulthandler
98
import functools
10-
import io
119
import itertools
1210
import json
1311
import math
@@ -23,14 +21,14 @@
2321
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
2422
from dataclasses import dataclass
2523
from pathlib import Path
26-
from typing import IO, TYPE_CHECKING, Any, Callable, Generator, Iterable, Literal, Sequence, TypeVar
24+
from typing import IO, TYPE_CHECKING, Any, Callable, Iterable, Literal, TypeVar
2725

2826
import numpy as np
2927
from sentencepiece import SentencePieceProcessor # type: ignore[import]
3028

3129
import os
3230
if 'NO_LOCAL_GGUF' not in os.environ:
33-
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
31+
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
3432
import gguf
3533

3634
if TYPE_CHECKING:
@@ -851,7 +849,7 @@ def add_meta_vocab(self, vocab: Vocab) -> None:
851849
elif isinstance(vocab, BpeVocab):
852850
self.gguf.add_tokenizer_model("gpt2")
853851
else:
854-
raise ValueError(f'Unknown vocab type: Not BpeVocab or SentencePieceVocab')
852+
raise ValueError('Unknown vocab type: Not BpeVocab or SentencePieceVocab')
855853
self.gguf.add_token_list(tokens)
856854
self.gguf.add_token_scores(scores)
857855
self.gguf.add_token_types(toktypes)
@@ -905,7 +903,7 @@ def maybe_do_quantize(item: tuple[DataType, NDArray]) -> NDArray:
905903
return dt.quantize(arr)
906904

907905
@staticmethod
908-
def write_all(fname_out: Path, ftype: GGMLFileType, params: Params, model: LazyModel, vocab: Vocab, svocab: gguf.SpecialVocab, concurrency: int = DEFAULT_CONCURRENCY, endianess=gguf.GGUFEndian.LITTLE) -> None:
906+
def write_all(fname_out: Path, ftype: GGMLFileType, params: Params, model: LazyModel, vocab: Vocab, svocab: gguf.SpecialVocab, concurrency: int = DEFAULT_CONCURRENCY, endianess: gguf.GGUFEndian = gguf.GGUFEndian.LITTLE) -> None:
909907
check_vocab_size(params, vocab)
910908

911909
of = OutputFile(fname_out, endianess=endianess)

‎examples/train-text-from-scratch/convert-train-checkpoint-to-gguf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from pathlib import Path
1010

1111
if 'NO_LOCAL_GGUF' not in os.environ:
12-
sys.path.insert(1, str(Path(__file__).parent / '..' / '..' / 'gguf-py' / 'gguf'))
12+
sys.path.insert(1, str(Path(__file__).parent / '..' / '..' / 'gguf-py'))
1313
import gguf
1414

1515
# gguf constants

‎gguf-py/gguf/__init__.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,5 @@
1-
from .gguf import *
1+
from .constants import *
2+
from .gguf_writer import *
3+
from .gguf_reader import *
4+
from .tensor_mapping import *
5+
from .vocab import *

0 commit comments

Comments
 (0)