Skip to content

Commit 2936dd6

Browse files
xaedesxaedes
committed
remove trailing whitespace
1 parent 7c8768f commit 2936dd6

File tree

4 files changed

+120
-120
lines changed

4 files changed

+120
-120
lines changed

‎examples/baby-llama/baby-llama.cpp

Lines changed: 61 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ void randomize_model(struct llama_model * model, int seed, float mean, float std
381381
randomize_tensor_normal(model->tok_embeddings, model->tok_embeddings->n_dims, model->tok_embeddings->ne, &rnd);
382382
randomize_tensor_normal(model->norm, model->norm->n_dims, model->norm->ne, &rnd);
383383
randomize_tensor_normal(model->output, model->output->n_dims, model->output->ne, &rnd);
384-
384+
385385
for (uint32_t i = 0; i < n_layer; ++i) {
386386
auto & layer = model->layers[i];
387387
randomize_tensor_normal(layer.attention_norm, layer.attention_norm->n_dims, layer.attention_norm->ne, &rnd);
@@ -415,7 +415,7 @@ void randomize_model_lora(struct llama_model_lora * model, int seed, float mean,
415415
randomize_tensor_normal(model->norm, model->norm->n_dims, model->norm->ne, &rnd);
416416
randomize_tensor_normal(model->outputa, model->outputa->n_dims, model->outputa->ne, &rnd);
417417
randomize_tensor_normal(model->outputb, model->outputb->n_dims, model->outputb->ne, &rnd);
418-
418+
419419
for (uint32_t i = 0; i < n_layer; ++i) {
420420
auto & layer = model->layers[i];
421421
randomize_tensor_normal(layer.attention_norm, layer.attention_norm->n_dims, layer.attention_norm->ne, &rnd);
@@ -508,14 +508,14 @@ bool init_kv_cache_lora(struct llama_kv_cache* cache, struct llama_model_lora *
508508
}
509509

510510
struct ggml_tensor * forward(
511-
struct llama_model * model,
512-
struct llama_kv_cache * cache,
511+
struct llama_model * model,
512+
struct llama_kv_cache * cache,
513513
struct ggml_context * ctx0,
514514
struct ggml_cgraph * gf,
515515
struct ggml_tensor * tokens_input,
516516
const int n_tokens,
517517
const int n_past) {
518-
518+
519519
const int N = n_tokens;
520520

521521
struct llama_kv_cache& kv_self = *cache;
@@ -569,11 +569,11 @@ struct ggml_tensor * forward(
569569
// Vcur shape [n_embd, N, 1, 1]
570570
struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, model->layers[il].wv, cur), n_embd, N)));
571571

572-
// kv_self.k shape [n_embd * n_ctx * n_layer, 1]
573-
// kv_self.v shape [n_embd * n_ctx * n_layer, 1]
572+
// kv_self.k shape [n_embd * n_ctx * n_layer, 1]
573+
// kv_self.v shape [n_embd * n_ctx * n_layer, 1]
574574
// k shape [n_embd * N, 1] == kv_self.k[:,n_past:n_past+N,il,0]
575575
// v shape [N, n_embd, 1, 1] == kv_self.v[:,n_past:n_past+N,il,0]
576-
576+
577577
/* {
578578
struct ggml_tensor * k = ggml_view_1d(ctx0, kv_self.k, N*n_embd, (ggml_element_size(kv_self.k)*n_embd)*(il*n_ctx + n_past));
579579
struct ggml_tensor * v = ggml_view_2d(ctx0, kv_self.v, N, n_embd,
@@ -597,7 +597,7 @@ struct ggml_tensor * forward(
597597
Qcur,
598598
0, 2, 1, 3);
599599

600-
// kv_self.k shape [n_embd * n_ctx * n_layer, 1]
600+
// kv_self.k shape [n_embd * n_ctx * n_layer, 1]
601601
// K shape [n_embd/n_head, n_past + N, n_head, 1]
602602
struct ggml_tensor * K =
603603
ggml_permute(ctx0,
@@ -641,7 +641,7 @@ struct ggml_tensor * forward(
641641
// KQV_merged = KQV.permute(0, 2, 1, 3)
642642
// KQV_merged shape [n_embd/n_head, n_head, N, 1]
643643
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
644-
// KQV_merged shape
644+
// KQV_merged shape
645645

646646
// cur = KQV_merged.contiguous().view(n_embd, N)
647647
// cur shape [n_embd,N,1,1]
@@ -734,14 +734,14 @@ struct ggml_tensor * forward(
734734

735735

736736
struct ggml_tensor * forward_lora(
737-
struct llama_model_lora * model,
738-
struct llama_kv_cache * cache,
737+
struct llama_model_lora * model,
738+
struct llama_kv_cache * cache,
739739
struct ggml_context * ctx0,
740740
struct ggml_cgraph * gf,
741741
struct ggml_tensor * tokens_input,
742742
const int n_tokens,
743743
const int n_past) {
744-
744+
745745
const int N = n_tokens;
746746

747747
struct llama_kv_cache& kv_self = *cache;
@@ -784,45 +784,45 @@ struct ggml_tensor * forward_lora(
784784
// wk shape [n_embd, n_embd, 1, 1]
785785
// Qcur shape [n_embd/n_head, n_head, N, 1]
786786
// Kcur shape [n_embd/n_head, n_head, N, 1]
787-
struct ggml_tensor * Qcur = ggml_rope(ctx0,
788-
ggml_reshape_3d(ctx0,
789-
ggml_mul_mat(ctx0,
790-
model->layers[il].wqa,
791-
ggml_mul_mat(ctx0,
792-
model->layers[il].wqb,
793-
cur)),
794-
n_embd/n_head, n_head, N),
787+
struct ggml_tensor * Qcur = ggml_rope(ctx0,
788+
ggml_reshape_3d(ctx0,
789+
ggml_mul_mat(ctx0,
790+
model->layers[il].wqa,
791+
ggml_mul_mat(ctx0,
792+
model->layers[il].wqb,
793+
cur)),
794+
n_embd/n_head, n_head, N),
795795
n_past, n_rot, 0);
796-
struct ggml_tensor * Kcur = ggml_rope(ctx0,
797-
ggml_reshape_3d(ctx0,
798-
ggml_mul_mat(ctx0,
799-
model->layers[il].wka,
800-
ggml_mul_mat(ctx0,
801-
model->layers[il].wkb,
802-
cur)),
803-
n_embd/n_head, n_head, N),
796+
struct ggml_tensor * Kcur = ggml_rope(ctx0,
797+
ggml_reshape_3d(ctx0,
798+
ggml_mul_mat(ctx0,
799+
model->layers[il].wka,
800+
ggml_mul_mat(ctx0,
801+
model->layers[il].wkb,
802+
cur)),
803+
n_embd/n_head, n_head, N),
804804
n_past, n_rot, 0);
805805

806806
// store key and value to memory
807807
{
808808
// compute the transposed [N, n_embd] V matrix
809809
// wv shape [n_embd, n_embd, 1, 1]
810810
// Vcur shape [n_embd, N, 1, 1]
811-
struct ggml_tensor * Vcur = ggml_cont(ctx0,
812-
ggml_transpose(ctx0,
813-
ggml_reshape_2d(ctx0,
814-
ggml_mul_mat(ctx0,
815-
model->layers[il].wva,
816-
ggml_mul_mat(ctx0,
817-
model->layers[il].wvb,
818-
cur)),
811+
struct ggml_tensor * Vcur = ggml_cont(ctx0,
812+
ggml_transpose(ctx0,
813+
ggml_reshape_2d(ctx0,
814+
ggml_mul_mat(ctx0,
815+
model->layers[il].wva,
816+
ggml_mul_mat(ctx0,
817+
model->layers[il].wvb,
818+
cur)),
819819
n_embd, N)));
820820

821-
// kv_self.k shape [n_embd * n_ctx * n_layer, 1]
822-
// kv_self.v shape [n_embd * n_ctx * n_layer, 1]
821+
// kv_self.k shape [n_embd * n_ctx * n_layer, 1]
822+
// kv_self.v shape [n_embd * n_ctx * n_layer, 1]
823823
// k shape [n_embd * N, 1] == kv_self.k[:,n_past:n_past+N,il,0]
824824
// v shape [N, n_embd, 1, 1] == kv_self.v[:,n_past:n_past+N,il,0]
825-
825+
826826
/* {
827827
struct ggml_tensor * k = ggml_view_1d(ctx0, kv_self.k, N*n_embd, (ggml_element_size(kv_self.k)*n_embd)*(il*n_ctx + n_past));
828828
struct ggml_tensor * v = ggml_view_2d(ctx0, kv_self.v, N, n_embd,
@@ -846,7 +846,7 @@ struct ggml_tensor * forward_lora(
846846
Qcur,
847847
0, 2, 1, 3);
848848

849-
// kv_self.k shape [n_embd * n_ctx * n_layer, 1]
849+
// kv_self.k shape [n_embd * n_ctx * n_layer, 1]
850850
// K shape [n_embd/n_head, n_past + N, n_head, 1]
851851
struct ggml_tensor * K =
852852
ggml_permute(ctx0,
@@ -890,7 +890,7 @@ struct ggml_tensor * forward_lora(
890890
// KQV_merged = KQV.permute(0, 2, 1, 3)
891891
// KQV_merged shape [n_embd/n_head, n_head, N, 1]
892892
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
893-
// KQV_merged shape
893+
// KQV_merged shape
894894

895895
// cur = KQV_merged.contiguous().view(n_embd, N)
896896
// cur shape [n_embd,N,1,1]
@@ -974,10 +974,10 @@ struct ggml_tensor * forward_lora(
974974

975975
// lm_head
976976
// inpL shape [n_vocab,N,1,1]
977-
inpL = ggml_mul_mat(ctx0,
978-
model->outputa,
979-
ggml_mul_mat(ctx0,
980-
model->outputb,
977+
inpL = ggml_mul_mat(ctx0,
978+
model->outputa,
979+
ggml_mul_mat(ctx0,
980+
model->outputb,
981981
inpL));
982982

983983
// ggml_set_scratch(ctx0, { 0, 0, nullptr, });
@@ -1094,12 +1094,12 @@ struct ggml_tensor * square_error_loss(struct ggml_context * ctx, struct ggml_te
10941094

10951095
struct ggml_tensor * cross_entropy_loss(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b) {
10961096
const float eps = 1e-3;
1097-
return
1098-
ggml_sum(ctx,
1099-
ggml_neg(ctx,
1100-
ggml_sum_rows(ctx,
1101-
ggml_mul(ctx,
1102-
ggml_soft_max(ctx, a),
1097+
return
1098+
ggml_sum(ctx,
1099+
ggml_neg(ctx,
1100+
ggml_sum_rows(ctx,
1101+
ggml_mul(ctx,
1102+
ggml_soft_max(ctx, a),
11031103
ggml_log(ctx,
11041104
ggml_add1(ctx,
11051105
ggml_soft_max(ctx, b),
@@ -1169,7 +1169,7 @@ int main(int argc, char ** argv) {
11691169
*/
11701170

11711171
// key + value cache for the self attention
1172-
struct llama_kv_cache kv_self;
1172+
struct llama_kv_cache kv_self;
11731173
printf("init_kv_cache\n");
11741174
kv_self.ctx = model.ctx;
11751175
init_kv_cache(&kv_self, &model);
@@ -1221,17 +1221,17 @@ int main(int argc, char ** argv) {
12211221
struct ggml_tensor * logits2 = forward(&model, &kv_self, ctx0, &gf, tokens_input2, n_tokens, n_past);
12221222
// struct ggml_tensor * logits3 = forward(&model, &kv_self, ctx0, &gf, tokens_input3, n_tokens, n_past);
12231223
// struct ggml_tensor * logits4 = forward(&model, &kv_self, ctx0, &gf, tokens_input4, n_tokens, n_past);
1224-
1224+
12251225
// struct ggml_tensor * e = cross_entropy_loss(ctx0, targets1, logits1);
12261226
// struct ggml_tensor * e = square_error_loss(ctx0, targets1, logits1);
1227-
1227+
12281228
struct ggml_tensor * e = ggml_add(ctx0,
12291229
square_error_loss(ctx0, targets1, logits1),
12301230
square_error_loss(ctx0, targets2, logits2));
12311231
// struct ggml_tensor * e = ggml_add(ctx0,
12321232
// cross_entropy_loss(ctx0, targets1, logits1),
12331233
// cross_entropy_loss(ctx0, targets2, logits2));
1234-
// struct ggml_tensor * e = ggml_add(ctx0,
1234+
// struct ggml_tensor * e = ggml_add(ctx0,
12351235
// ggml_add(ctx0,
12361236
// cross_entropy_loss(ctx0, targets1, logits1),
12371237
// cross_entropy_loss(ctx0, targets2, logits2)),
@@ -1260,7 +1260,7 @@ int main(int argc, char ** argv) {
12601260
opt_params_lbfgs.lbfgs.n_iter = 16;
12611261
// ggml_opt(ctx0, opt_params_adam, e);
12621262
ggml_opt(ctx0, opt_params_lbfgs, e);
1263-
//
1263+
//
12641264
ggml_build_forward_expand(&gf, e);
12651265
ggml_graph_compute(ctx0, &gf);
12661266

@@ -1292,7 +1292,7 @@ int main(int argc, char ** argv) {
12921292

12931293
struct ggml_tensor * tokens_input = ggml_new_tensor_1d(model.ctx, GGML_TYPE_I32, n_tokens);
12941294
struct ggml_tensor * targets = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, n_vocab, n_tokens);
1295-
1295+
12961296
get_example_targets(137, tokens_input, targets);
12971297
for (int i=sample_ctx; i<n_tokens; ++i) {
12981298
ggml_set_i32_1d(tokens_input, i, n_vocab/2);
@@ -1327,14 +1327,14 @@ int main(int argc, char ** argv) {
13271327

13281328
// int sample_at = n_tokens-1;
13291329
int token = ggml_get_i32_1d(best_samples, sample_ctx-1);
1330-
1330+
13311331
// print_row(probs, sample_at);
13321332
print_token(token, n_vocab);
13331333

13341334
lshift_examples(tokens_input, targets, 1);
13351335
ggml_set_i32_1d(tokens_input, 0, 0);
13361336
ggml_set_i32_1d(tokens_input, sample_ctx-1, token);
1337-
1337+
13381338
// printf("---\n");
13391339
// for (int i=0; i<sample_ctx-1; ++i) {
13401340
// print_token(ggml_get_i32_1d(tokens_input, i), model.hparams.n_vocab);
@@ -1350,7 +1350,7 @@ int main(int argc, char ** argv) {
13501350
}
13511351
printf("important (dont optimize it away, compiler!) : %d\n", important_sum);
13521352
}
1353-
1353+
13541354
print_matrix(model.tok_embeddings);
13551355

13561356
printf("done\n");

0 commit comments

Comments
 (0)