Skip to content

Commit 82b2404

Browse files
compiladecompilade
committed
Merge branch 'master' into compilade/bitnet-ternary
2 parents 69f7726 + 06943a6 commit 82b2404

File tree

22 files changed

+75
-58
lines changed

22 files changed

+75
-58
lines changed

‎.github/workflows/bench.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,8 @@ jobs:
129129
130130
- name: Server bench
131131
id: server_bench
132+
env:
133+
HEAD_REF: ${{ github.head_ref || github.ref_name }}
132134
run: |
133135
set -eux
134136
@@ -137,7 +139,7 @@ jobs:
137139
python bench.py \
138140
--runner-label ${{ env.RUNNER_LABEL }} \
139141
--name ${{ github.job }} \
140-
--branch ${{ github.head_ref || github.ref_name }} \
142+
--branch $HEAD_REF \
141143
--commit ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha }} \
142144
--scenario script.js \
143145
--duration ${{ github.event.inputs.duration || env.DURATION }} \

‎.github/workflows/build.yml

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
sysctl -a
4848
mkdir build
4949
cd build
50-
cmake -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF ..
50+
cmake -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF ..
5151
cmake --build . --config Release -j $(sysctl -n hw.logicalcpu)
5252
5353
- name: Test
@@ -105,7 +105,7 @@ jobs:
105105
sysctl -a
106106
# Metal is disabled due to intermittent failures with Github runners not having a GPU:
107107
# https://github.com/ggerganov/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
108-
cmake -B build -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL=OFF -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF
108+
cmake -B build -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL=OFF -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF
109109
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
110110
111111
- name: Test
@@ -222,7 +222,7 @@ jobs:
222222
run: |
223223
mkdir build
224224
cd build
225-
cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF
225+
cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF
226226
cmake --build . --config Release -j $(nproc)
227227
228228
- name: Test
@@ -696,22 +696,20 @@ jobs:
696696
strategy:
697697
matrix:
698698
include:
699-
- build: 'rpc-x64'
700-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=ON'
701699
- build: 'noavx-x64'
702-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DBUILD_SHARED_LIBS=ON'
700+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DBUILD_SHARED_LIBS=ON'
703701
- build: 'avx2-x64'
704-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON'
702+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=ON'
705703
- build: 'avx-x64'
706-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX2=OFF -DBUILD_SHARED_LIBS=ON'
704+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX2=OFF -DBUILD_SHARED_LIBS=ON'
707705
- build: 'avx512-x64'
708-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
706+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
709707
- build: 'openblas-x64'
710-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_BLAS=ON -DBUILD_SHARED_LIBS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
708+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BLAS=ON -DBUILD_SHARED_LIBS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
711709
- build: 'kompute-x64'
712-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON'
710+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON'
713711
- build: 'vulkan-x64'
714-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_VULKAN=ON -DBUILD_SHARED_LIBS=ON'
712+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_VULKAN=ON -DBUILD_SHARED_LIBS=ON'
715713
- build: 'llvm-arm64'
716714
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON'
717715
- build: 'msvc-arm64'

‎.github/workflows/python-check-requirements.yml

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,13 @@ on:
66
- '.github/workflows/python-check-requirements.yml'
77
- 'scripts/check-requirements.sh'
88
- 'convert*.py'
9-
- 'requirements.txt'
10-
- 'requirements/*.txt'
9+
- '**/requirements*.txt'
1110
pull_request:
1211
paths:
1312
- '.github/workflows/python-check-requirements.yml'
1413
- 'scripts/check-requirements.sh'
1514
- 'convert*.py'
16-
- 'requirements.txt'
17-
- 'requirements/*.txt'
15+
- '**/requirements*.txt'
1816

1917
concurrency:
2018
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,12 @@ Unless otherwise noted these projects are open-source with permissive licensing:
186186

187187
- [akx/ggify](https://github.com/akx/ggify) – download PyTorch models from HuggingFace Hub and convert them to GGML
188188
- [crashr/gppm](https://github.com/crashr/gppm) – launch llama.cpp instances utilizing NVIDIA Tesla P40 or P100 GPUs with reduced idle power consumption
189+
- [gpustack/gguf-parser](https://github.com/gpustack/gguf-parser-go/tree/main/cmd/gguf-parser) - review/check the GGUF file and estimate the memory usage
189190

190191
**Infrastructure:**
191192

192193
- [Paddler](https://github.com/distantmagic/paddler) - Stateful load balancer custom-tailored for llama.cpp
194+
- [GPUStack](https://github.com/gpustack/gpustack) - Manage GPU clusters for running LLMs
193195

194196
**Games:**
195197
- [Lucy's Labyrinth](https://github.com/MorganRO8/Lucys_Labyrinth) - A simple maze game where agents controlled by an AI model will try to trick you.

‎common/grammar-parser.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,9 @@ namespace grammar_parser {
369369
}
370370
// Validate the state to ensure that all rules are defined
371371
for (const auto & rule : state.rules) {
372+
if (rule.empty()) {
373+
throw std::runtime_error("Undefined rule");
374+
}
372375
for (const auto & elem : rule) {
373376
if (elem.type == LLAMA_GRETYPE_RULE_REF) {
374377
// Ensure that the rule at that location exists

‎examples/export-lora/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ For example:
1717

1818
```bash
1919
./bin/llama-export-lora \
20-
-m open-llama-3b-v2-q8_0.gguf \
21-
-o open-llama-3b-v2-q8_0-english2tokipona-chat.gguf \
22-
--lora lora-open-llama-3b-v2-q8_0-english2tokipona-chat-LATEST.gguf
20+
-m open-llama-3b-v2.gguf \
21+
-o open-llama-3b-v2-english2tokipona-chat.gguf \
22+
--lora lora-open-llama-3b-v2-english2tokipona-chat-LATEST.gguf
2323
```
2424

2525
Multiple LORA adapters can be applied by passing multiple `--lora FNAME` or `--lora-scaled FNAME S` command line parameters:

‎examples/export-lora/export-lora.cpp

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@
1010

1111
static bool g_verbose = false;
1212

13+
struct tensor_transformation {
14+
struct ggml_tensor * in;
15+
struct ggml_tensor * out;
16+
bool is_copy;
17+
};
18+
1319
static std::string get_kv_str(struct gguf_context * ctx_gguf, const std::string & key){
1420
int id = gguf_find_key(ctx_gguf, key.c_str());
1521
return id < 0 ? "" : std::string(gguf_get_val_str(ctx_gguf, id));
@@ -198,8 +204,7 @@ struct lora_merge_ctx {
198204
}
199205

200206
// mapping base tensor to out tensor (same shape with base, but different type)
201-
// if out_tensor == nullptr, we only copy it
202-
std::vector<std::pair<struct ggml_tensor *, struct ggml_tensor *>> base_to_out_tensors;
207+
std::vector<tensor_transformation> trans;
203208
for (auto & it : base_model.tensors) {
204209
bool t_a = true;
205210
bool t_b = true;
@@ -212,14 +217,22 @@ struct lora_merge_ctx {
212217
// only copy
213218
struct ggml_tensor * cpy_tensor = ggml_dup_tensor(ctx_out_ggml, base_tensor);
214219
ggml_set_name(cpy_tensor, base_tensor->name);
215-
base_to_out_tensors.push_back(std::make_pair(cpy_tensor, nullptr));
220+
trans.push_back({
221+
cpy_tensor,
222+
cpy_tensor,
223+
true,
224+
});
216225
gguf_add_tensor(ctx_out, cpy_tensor);
217226
} else if (t_a && t_b) {
218227
// need merging
219228
struct ggml_tensor * out_tensor = ggml_new_tensor(
220229
ctx_out_ggml, get_out_tensor_type(base_tensor), GGML_MAX_DIMS, base_tensor->ne);
221230
ggml_set_name(out_tensor, base_tensor->name);
222-
base_to_out_tensors.push_back(std::make_pair(base_tensor, out_tensor));
231+
trans.push_back({
232+
base_tensor,
233+
out_tensor,
234+
false,
235+
});
223236
gguf_add_tensor(ctx_out, out_tensor);
224237
} else {
225238
throw std::runtime_error("tensor " + it.first + " missing either lora_a or lora_b");
@@ -234,12 +247,12 @@ struct lora_merge_ctx {
234247

235248
// process base model tensors
236249
size_t n_merged = 0;
237-
for (auto & it : base_to_out_tensors) {
238-
if (it.second != nullptr) {
239-
merge_tensor(it.first, it.second);
250+
for (auto & it : trans) {
251+
if (!it.is_copy) {
252+
merge_tensor(it.in, it.out);
240253
n_merged++;
241254
} else {
242-
copy_tensor(it.first);
255+
copy_tensor(it.in);
243256
}
244257
}
245258

@@ -252,7 +265,7 @@ struct lora_merge_ctx {
252265
}
253266

254267
printf("%s : merged %ld tensors with lora adapters\n", __func__, n_merged);
255-
printf("%s : wrote %ld tensors to output file\n", __func__, base_to_out_tensors.size());
268+
printf("%s : wrote %ld tensors to output file\n", __func__, trans.size());
256269
}
257270

258271
void copy_tensor(struct ggml_tensor * base) {
@@ -285,6 +298,10 @@ struct lora_merge_ctx {
285298
for (size_t i = 0; i < adapters.size(); ++i) {
286299
auto t_a = adapters[i]->get_tensor(name_lora_a);
287300
auto t_b = adapters[i]->get_tensor(name_lora_b);
301+
// TODO: add support for quantized lora
302+
if (ggml_is_quantized(t_a->type) || ggml_is_quantized(t_b->type)) {
303+
throw std::runtime_error("quantized LoRA adapters is not supported, please retry with f16 or f32");
304+
}
288305
inp_a[i] = ggml_dup_tensor(ctx, t_a);
289306
inp_b[i] = ggml_dup_tensor(ctx, t_b);
290307
}

‎examples/llava/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
--extra-index-url https://download.pytorch.org/whl/cpu
33
pillow~=10.2.0
44
torch~=2.2.1
5-
torchvision==0.17.1
5+
torchvision~=0.17.1

‎examples/server/server.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -631,6 +631,7 @@ struct server_context {
631631

632632
bool clean_kv_cache = true;
633633
bool add_bos_token = true;
634+
bool has_eos_token = false;
634635

635636
int32_t n_ctx; // total context for all clients / slots
636637

@@ -693,7 +694,7 @@ struct server_context {
693694
n_ctx = llama_n_ctx(ctx);
694695

695696
add_bos_token = llama_should_add_bos_token(model);
696-
GGML_ASSERT(llama_add_eos_token(model) != 1);
697+
has_eos_token = llama_add_eos_token(model) != 1;
697698

698699
return true;
699700
}
@@ -1031,7 +1032,7 @@ struct server_context {
10311032
{
10321033
slot.sparams.logit_bias.clear();
10331034

1034-
if (json_value(data, "ignore_eos", false)) {
1035+
if (json_value(data, "ignore_eos", false) && has_eos_token) {
10351036
slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
10361037
}
10371038

‎ggml/include/ggml.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,8 @@
244244
#define GGML_EXIT_SUCCESS 0
245245
#define GGML_EXIT_ABORTED 1
246246

247+
#define GGML_ROPE_TYPE_NEOX 2
248+
247249
#define GGUF_MAGIC "GGUF"
248250

249251
#define GGUF_VERSION 3
@@ -1455,8 +1457,8 @@ extern "C" {
14551457
struct ggml_tensor * b);
14561458

14571459
// rotary position embedding
1458-
// if mode & 1 == 1, skip n_past elements (NOT SUPPORTED)
1459-
// if mode & 2 == 1, GPT-NeoX style
1460+
// if (mode & 1) - skip n_past elements (NOT SUPPORTED)
1461+
// if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style
14601462
//
14611463
// b is an int32 vector with size a->ne[2], it contains the positions
14621464
GGML_API struct ggml_tensor * ggml_rope(

‎ggml/src/ggml-cann/aclnn_ops.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2881,7 +2881,7 @@ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
28812881
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast,
28822882
beta_slow, corr_dims);
28832883

2884-
const bool is_neox = mode & 2;
2884+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
28852885

28862886
// init cos/sin cache
28872887
ggml_cann_pool_alloc sin_allocator(

‎ggml/src/ggml-cuda/rope.cu

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
226226
memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
227227
memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
228228

229-
const bool is_neox = mode & 2;
229+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
230230

231231
const int32_t * pos = (const int32_t *) src1_d;
232232

‎ggml/src/ggml-metal.m

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2313,7 +2313,7 @@ static enum ggml_status ggml_metal_graph_compute(
23132313
memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
23142314
memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
23152315

2316-
const bool is_neox = mode & 2;
2316+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
23172317

23182318
id<MTLComputePipelineState> pipeline = nil;
23192319

‎ggml/src/ggml-sycl/rope.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ void ggml_sycl_op_rope(
226226
memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
227227
memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
228228

229-
const bool is_neox = mode & 2;
229+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
230230

231231
const int32_t * pos = (const int32_t *) src1_dd;
232232

‎ggml/src/ggml-vulkan.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4053,7 +4053,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
40534053
case GGML_OP_ROPE:
40544054
{
40554055
const int mode = ((const int32_t *) dst->op_params)[2];
4056-
const bool is_neox = mode & 2;
4056+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
40574057

40584058
if (is_neox) {
40594059
if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {

‎ggml/src/ggml.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14131,7 +14131,7 @@ static void ggml_compute_forward_rope_f32(
1413114131
float corr_dims[2];
1413214132
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
1413314133

14134-
const bool is_neox = mode & 2;
14134+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
1413514135

1413614136
const float * freq_factors = NULL;
1413714137
if (src2 != NULL) {
@@ -14256,7 +14256,7 @@ static void ggml_compute_forward_rope_f16(
1425614256
float corr_dims[2];
1425714257
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
1425814258

14259-
const bool is_neox = mode & 2;
14259+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
1426014260

1426114261
const float * freq_factors = NULL;
1426214262
if (src2 != NULL) {
@@ -21168,7 +21168,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
2116821168
(int64_t) info->ne[2] *
2116921169
(int64_t) info->ne[3];
2117021170

21171-
if (ne % ggml_blck_size(info->type) != 0) {
21171+
if (ggml_blck_size(info->type) == 0 || ne % ggml_blck_size(info->type) != 0) {
2117221172
fprintf(stderr, "%s: tensor '%s' of type %d (%s) number of elements (%" PRId64 ") is not a multiple of block size (%" PRId64 ")\n",
2117321173
__func__, info->name.data, (int) info->type, ggml_type_name(info->type), ne, ggml_blck_size(info->type));
2117421174
fclose(file);

‎ggml/src/kompute-shaders/op_rope_f16.comp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ void main() {
1111
const uint i2 = gl_WorkGroupID.y;
1212
const uint i1 = gl_WorkGroupID.x;
1313

14-
const bool is_neox = (pcs.mode & 2) != 0;
14+
const bool is_neox = (pcs.mode & GGML_ROPE_TYPE_NEOX) != 0;
1515

1616
float corr_dims[2];
1717
rope_yarn_corr_dims(pcs.n_dims, pcs.n_ctx_orig, pcs.freq_base, pcs.beta_fast, pcs.beta_slow, corr_dims);

‎ggml/src/kompute-shaders/op_rope_f32.comp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ void main() {
1111
const uint i2 = gl_WorkGroupID.y;
1212
const uint i1 = gl_WorkGroupID.x;
1313

14-
const bool is_neox = (pcs.mode & 2) != 0;
14+
const bool is_neox = (pcs.mode & GGML_ROPE_TYPE_NEOX) != 0;
1515

1616
float corr_dims[2];
1717
rope_yarn_corr_dims(pcs.n_dims, pcs.n_ctx_orig, pcs.freq_base, pcs.beta_fast, pcs.beta_slow, corr_dims);

‎ggml/src/kompute-shaders/rope_common.comp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#include "common.comp"
22

3+
#define GGML_ROPE_TYPE_NEOX 2
4+
35
// TODO: use a local size of 32 or more (Metal uses 1024)
46
layout(local_size_x = 1) in;
57

‎include/llama.h

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,10 @@ extern "C" {
9595
LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,
9696
};
9797

98-
// note: these values should be synchronized with ggml_rope
99-
// TODO: maybe move this enum to ggml.h (ggml_rope_type)
10098
enum llama_rope_type {
10199
LLAMA_ROPE_TYPE_NONE = -1,
102-
LLAMA_ROPE_TYPE_NORM = 0,
103-
LLAMA_ROPE_TYPE_NEOX = 2,
104-
LLAMA_ROPE_TYPE_GLM = 4,
100+
LLAMA_ROPE_TYPE_NORM = 0,
101+
LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,
105102
};
106103

107104
enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file

‎src/llama-sampling.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,14 +85,14 @@ void llama_sample_top_k_impl(struct llama_sampling * smpl, llama_token_data_arra
8585
constexpr float bucket_low = -10.0f;
8686
constexpr float bucket_high = 10.0f;
8787
constexpr float bucket_scale = nbuckets/(bucket_high - bucket_low);
88-
constexpr float bucker_inter = -bucket_low * bucket_scale;
88+
constexpr float bucket_inter = -bucket_low * bucket_scale;
8989

9090
std::vector<int> bucket_idx(candidates->size);
9191
std::vector<int> histo(nbuckets, 0);
9292

9393
for (int i = 0; i < (int)candidates->size; ++i) {
9494
const float val = candidates->data[i].logit;
95-
int ib = int(bucket_scale * val + bucker_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
95+
int ib = int(bucket_scale * val + bucket_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
9696
ib = std::max(0, std::min(nbuckets-1, ib));
9797
bucket_idx[i] = ib;
9898
++histo[ib];

0 commit comments

Comments
 (0)