Skip to content

Commit 5360799

Browse files
mofosynemofosyne
committed
llama-gguf-hash: add sha256
1 parent ccc6ef7 commit 5360799

File tree

9 files changed

+342
-20
lines changed

9 files changed

+342
-20
lines changed

‎Makefile

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1181,13 +1181,17 @@ llama-gguf: examples/gguf/gguf.cpp \
11811181

11821182
examples/gguf-hash/deps/sha1/sha1.o: \
11831183
examples/gguf-hash/deps/sha1/sha1.c
1184-
$(CC) $(CFLAGS) -c $< -o $@
1184+
$(CC) $(CFLAGS) -Iexamples/gguf-hash/deps -c $< -o $@
11851185

11861186
examples/gguf-hash/deps/xxhash/xxhash.o: \
11871187
examples/gguf-hash/deps/xxhash/xxhash.c
1188-
$(CC) $(CFLAGS) -c $< -o $@
1188+
$(CC) $(CFLAGS) -Iexamples/gguf-hash/deps -c $< -o $@
1189+
1190+
examples/gguf-hash/deps/sha256/sha256.o: \
1191+
examples/gguf-hash/deps/sha256/sha256.c
1192+
$(CC) $(CFLAGS) -Iexamples/gguf-hash/deps -c $< -o $@
11891193

1190-
llama-gguf-hash: examples/gguf-hash/gguf-hash.cpp examples/gguf-hash/deps/sha1/sha1.o examples/gguf-hash/deps/xxhash/xxhash.o\
1194+
llama-gguf-hash: examples/gguf-hash/gguf-hash.cpp examples/gguf-hash/deps/sha1/sha1.o examples/gguf-hash/deps/xxhash/xxhash.o examples/gguf-hash/deps/sha256/sha256.o\
11911195
$(OBJ_ALL)
11921196
$(CXX) $(CXXFLAGS) -Iexamples/gguf-hash/deps -c $< -o $(call GET_OBJ_FILE, $<)
11931197
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)

‎examples/gguf-hash/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ add_library(xxhash OBJECT deps/xxhash/xxhash.c deps/xxhash/xxhash.h)
88
target_link_libraries(${TARGET} PRIVATE xxhash)
99
add_library(sha1 OBJECT deps/sha1/sha1.c deps/sha1/sha1.h)
1010
target_link_libraries(${TARGET} PRIVATE sha1)
11+
add_library(sha256 OBJECT deps/sha256/sha256.c deps/sha256/sha256.h)
12+
target_link_libraries(${TARGET} PRIVATE sha256)
1113

1214
target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT})
1315
target_compile_features(${TARGET} PRIVATE cxx_std_11)

‎examples/gguf-hash/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ CLI to hash GGUF files.
77
- `--xxhash`: use xhash (default)
88
- `--sha1`: use sha1
99
- `--uuid`: use uuid
10+
- `--sha256`: use sha256
1011

1112
### Compile Example
1213

@@ -18,6 +19,7 @@ make -C build llama-gguf-hash VERBOSE=1
1819
./build/bin/llama-gguf-hash --xxhash test.gguf
1920
./build/bin/llama-gguf-hash --sha1 test.gguf
2021
./build/bin/llama-gguf-hash --uuid test.gguf
22+
./build/bin/llama-gguf-hash --sha256 test.gguf
2123
```
2224

2325
### Crypto/Hash Libraries Used
@@ -26,3 +28,4 @@ These micro c libraries dependencies was installed via the [clib c package manag
2628

2729
- https://github.com/mofosyne/xxHash
2830
- https://github.com/clibs/sha1/
31+
- https://github.com/jb55/sha256.c

‎examples/gguf-hash/deps/rotate-bits/package.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"name": "rotate-bits",
3+
"version": "0.1.1",
4+
"repo": "jb55/rotate-bits.h",
5+
"description": "rotate bits",
6+
"keywords": ["rotl", "rotr"],
7+
"src": ["rotate-bits.h"],
8+
"license": "Public Domain",
9+
"development": {
10+
"thlorenz/tap.c": "*"
11+
}
12+
}
13+

‎examples/gguf-hash/deps/rotate-bits/rotate-bits.h

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
2+
3+
#ifndef __ROTATE_DEFS_H
4+
#define __ROTATE_DEFS_H
5+
6+
#ifdef _MSC_VER
7+
8+
#include <stdlib.h>
9+
10+
#define ROTL32(v, n) _rotl((v), (n))
11+
#define ROTL64(v, n) _rotl64((v), (n))
12+
13+
#define ROTR32(v, n) _rotr((v), (n))
14+
#define ROTR64(v, n) _rotr64((v), (n))
15+
16+
#else
17+
18+
#include <stdint.h>
19+
20+
#define U8V(v) ((uint8_t)(v) & 0xFFU)
21+
#define U16V(v) ((uint16_t)(v) & 0xFFFFU)
22+
#define U32V(v) ((uint32_t)(v) & 0xFFFFFFFFU)
23+
#define U64V(v) ((uint64_t)(v) & 0xFFFFFFFFFFFFFFFFU)
24+
25+
#define ROTL32(v, n) \
26+
(U32V((uint32_t)(v) << (n)) | ((uint32_t)(v) >> (32 - (n))))
27+
28+
// tests fail if we don't have this cast...
29+
#define ROTL64(v, n) \
30+
(U64V((uint64_t)(v) << (n)) | ((uint64_t)(v) >> (64 - (n))))
31+
32+
#define ROTR32(v, n) ROTL32(v, 32 - (n))
33+
#define ROTR64(v, n) ROTL64(v, 64 - (n))
34+
35+
#endif
36+
37+
#define ROTL8(v, n) \
38+
(U8V((uint8_t)(v) << (n)) | ((uint8_t)(v) >> (8 - (n))))
39+
40+
#define ROTL16(v, n) \
41+
(U16V((uint16_t)(v) << (n)) | ((uint16_t)(v) >> (16 - (n))))
42+
43+
#define ROTR8(v, n) ROTL8(v, 8 - (n))
44+
#define ROTR16(v, n) ROTL16(v, 16 - (n))
45+
46+
#endif

‎examples/gguf-hash/deps/sha256/package.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"name": "sha256",
3+
"version": "0.0.2",
4+
"repo": "jb55/sha256.c",
5+
"description": "sha256 in c",
6+
"keywords": ["sha256", "sha2"],
7+
"src": ["sha256.c", "sha256.h"],
8+
"dependencies": {
9+
"jb55/rotate-bits.h": "0.1.1"
10+
},
11+
"development": {
12+
"thlorenz/tap.c": "*"
13+
}
14+
}
15+

‎examples/gguf-hash/deps/sha256/sha256.c

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
/* Crypto/Sha256.c -- SHA-256 Hash
2+
2010-06-11 : Igor Pavlov : Public domain
3+
This code is based on public domain code from Wei Dai's Crypto++ library. */
4+
5+
#include "rotate-bits/rotate-bits.h"
6+
#include "sha256.h"
7+
8+
/* define it for speed optimization */
9+
#define _SHA256_UNROLL
10+
#define _SHA256_UNROLL2
11+
12+
void
13+
sha256_init(sha256_t *p)
14+
{
15+
p->state[0] = 0x6a09e667;
16+
p->state[1] = 0xbb67ae85;
17+
p->state[2] = 0x3c6ef372;
18+
p->state[3] = 0xa54ff53a;
19+
p->state[4] = 0x510e527f;
20+
p->state[5] = 0x9b05688c;
21+
p->state[6] = 0x1f83d9ab;
22+
p->state[7] = 0x5be0cd19;
23+
p->count = 0;
24+
}
25+
26+
#define S0(x) (ROTR32(x, 2) ^ ROTR32(x,13) ^ ROTR32(x, 22))
27+
#define S1(x) (ROTR32(x, 6) ^ ROTR32(x,11) ^ ROTR32(x, 25))
28+
#define s0(x) (ROTR32(x, 7) ^ ROTR32(x,18) ^ (x >> 3))
29+
#define s1(x) (ROTR32(x,17) ^ ROTR32(x,19) ^ (x >> 10))
30+
31+
#define blk0(i) (W[i] = data[i])
32+
#define blk2(i) (W[i&15] += s1(W[(i-2)&15]) + W[(i-7)&15] + s0(W[(i-15)&15]))
33+
34+
#define Ch(x,y,z) (z^(x&(y^z)))
35+
#define Maj(x,y,z) ((x&y)|(z&(x|y)))
36+
37+
#define a(i) T[(0-(i))&7]
38+
#define b(i) T[(1-(i))&7]
39+
#define c(i) T[(2-(i))&7]
40+
#define d(i) T[(3-(i))&7]
41+
#define e(i) T[(4-(i))&7]
42+
#define f(i) T[(5-(i))&7]
43+
#define g(i) T[(6-(i))&7]
44+
#define h(i) T[(7-(i))&7]
45+
46+
47+
#ifdef _SHA256_UNROLL2
48+
49+
#define R(a,b,c,d,e,f,g,h, i) h += S1(e) + Ch(e,f,g) + K[i+j] + (j?blk2(i):blk0(i));\
50+
d += h; h += S0(a) + Maj(a, b, c)
51+
52+
#define RX_8(i) \
53+
R(a,b,c,d,e,f,g,h, i); \
54+
R(h,a,b,c,d,e,f,g, (i+1)); \
55+
R(g,h,a,b,c,d,e,f, (i+2)); \
56+
R(f,g,h,a,b,c,d,e, (i+3)); \
57+
R(e,f,g,h,a,b,c,d, (i+4)); \
58+
R(d,e,f,g,h,a,b,c, (i+5)); \
59+
R(c,d,e,f,g,h,a,b, (i+6)); \
60+
R(b,c,d,e,f,g,h,a, (i+7))
61+
62+
#else
63+
64+
#define R(i) h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[i+j] + (j?blk2(i):blk0(i));\
65+
d(i) += h(i); h(i) += S0(a(i)) + Maj(a(i), b(i), c(i))
66+
67+
#ifdef _SHA256_UNROLL
68+
69+
#define RX_8(i) R(i+0); R(i+1); R(i+2); R(i+3); R(i+4); R(i+5); R(i+6); R(i+7);
70+
71+
#endif
72+
73+
#endif
74+
75+
static const uint32_t K[64] = {
76+
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
77+
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
78+
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
79+
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
80+
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
81+
0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
82+
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
83+
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
84+
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
85+
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
86+
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
87+
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
88+
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
89+
0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
90+
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
91+
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
92+
};
93+
94+
static void
95+
sha256_transform(uint32_t *state, const uint32_t *data)
96+
{
97+
uint32_t W[16];
98+
unsigned j;
99+
#ifdef _SHA256_UNROLL2
100+
uint32_t a,b,c,d,e,f,g,h;
101+
a = state[0];
102+
b = state[1];
103+
c = state[2];
104+
d = state[3];
105+
e = state[4];
106+
f = state[5];
107+
g = state[6];
108+
h = state[7];
109+
#else
110+
uint32_t T[8];
111+
for (j = 0; j < 8; j++)
112+
T[j] = state[j];
113+
#endif
114+
115+
for (j = 0; j < 64; j += 16)
116+
{
117+
#if defined(_SHA256_UNROLL) || defined(_SHA256_UNROLL2)
118+
RX_8(0); RX_8(8);
119+
#else
120+
unsigned i;
121+
for (i = 0; i < 16; i++) { R(i); }
122+
#endif
123+
}
124+
125+
#ifdef _SHA256_UNROLL2
126+
state[0] += a;
127+
state[1] += b;
128+
state[2] += c;
129+
state[3] += d;
130+
state[4] += e;
131+
state[5] += f;
132+
state[6] += g;
133+
state[7] += h;
134+
#else
135+
for (j = 0; j < 8; j++)
136+
state[j] += T[j];
137+
#endif
138+
139+
/* Wipe variables */
140+
/* memset(W, 0, sizeof(W)); */
141+
/* memset(T, 0, sizeof(T)); */
142+
}
143+
144+
#undef S0
145+
#undef S1
146+
#undef s0
147+
#undef s1
148+
149+
static void
150+
sha256_write_byte_block(sha256_t *p)
151+
{
152+
uint32_t data32[16];
153+
unsigned i;
154+
for (i = 0; i < 16; i++)
155+
data32[i] =
156+
((uint32_t)(p->buffer[i * 4 ]) << 24) +
157+
((uint32_t)(p->buffer[i * 4 + 1]) << 16) +
158+
((uint32_t)(p->buffer[i * 4 + 2]) << 8) +
159+
((uint32_t)(p->buffer[i * 4 + 3]));
160+
sha256_transform(p->state, data32);
161+
}
162+
163+
164+
void
165+
sha256_hash(unsigned char *buf, const unsigned char *data, size_t size)
166+
{
167+
sha256_t hash;
168+
sha256_init(&hash);
169+
sha256_update(&hash, data, size);
170+
sha256_final(&hash, buf);
171+
}
172+
173+
174+
void
175+
sha256_update(sha256_t *p, const unsigned char *data, size_t size)
176+
{
177+
uint32_t curBufferPos = (uint32_t)p->count & 0x3F;
178+
while (size > 0)
179+
{
180+
p->buffer[curBufferPos++] = *data++;
181+
p->count++;
182+
size--;
183+
if (curBufferPos == 64)
184+
{
185+
curBufferPos = 0;
186+
sha256_write_byte_block(p);
187+
}
188+
}
189+
}
190+
191+
192+
void
193+
sha256_final(sha256_t *p, unsigned char *digest)
194+
{
195+
uint64_t lenInBits = (p->count << 3);
196+
uint32_t curBufferPos = (uint32_t)p->count & 0x3F;
197+
unsigned i;
198+
p->buffer[curBufferPos++] = 0x80;
199+
while (curBufferPos != (64 - 8))
200+
{
201+
curBufferPos &= 0x3F;
202+
if (curBufferPos == 0)
203+
sha256_write_byte_block(p);
204+
p->buffer[curBufferPos++] = 0;
205+
}
206+
for (i = 0; i < 8; i++)
207+
{
208+
p->buffer[curBufferPos++] = (unsigned char)(lenInBits >> 56);
209+
lenInBits <<= 8;
210+
}
211+
sha256_write_byte_block(p);
212+
213+
for (i = 0; i < 8; i++)
214+
{
215+
*digest++ = (unsigned char)(p->state[i] >> 24);
216+
*digest++ = (unsigned char)(p->state[i] >> 16);
217+
*digest++ = (unsigned char)(p->state[i] >> 8);
218+
*digest++ = (unsigned char)(p->state[i]);
219+
}
220+
sha256_init(p);
221+
}

‎examples/gguf-hash/deps/sha256/sha256.h

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/* Sha256.h -- SHA-256 Hash
2+
2010-06-11 : Igor Pavlov : Public domain */
3+
4+
#ifndef __CRYPTO_SHA256_H
5+
#define __CRYPTO_SHA256_H
6+
7+
#include <stdlib.h>
8+
#include <stdint.h>
9+
10+
#define SHA256_DIGEST_SIZE 32
11+
12+
typedef struct sha256_t
13+
{
14+
uint32_t state[8];
15+
uint64_t count;
16+
unsigned char buffer[64];
17+
} sha256_t;
18+
19+
void sha256_init(sha256_t *p);
20+
void sha256_update(sha256_t *p, const unsigned char *data, size_t size);
21+
void sha256_final(sha256_t *p, unsigned char *digest);
22+
void sha256_hash(unsigned char *buf, const unsigned char *data, size_t size);
23+
24+
#endif

0 commit comments

Comments
 (0)