Skip to content

Commit 49d6daa

Browse files
committed
vastly improve training results
instead of logit targets 0 and 1 use -1 and +1.
1 parent 93201ab commit 49d6daa

File tree

1 file changed

+2
-1
lines changed

1 file changed

+2
-1
lines changed

examples/baby-llama/baby-llama.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1058,7 +1058,8 @@ void get_example_targets(int example_id, struct ggml_tensor * tokens_input, stru
10581058
int n_tokens = tokens_input->ne[0];
10591059
int n_vocab = targets->ne[0];
10601060
float randomness = 0.0f;
1061-
ggml_set_zero(targets);
1061+
// ggml_set_zero(targets);
1062+
ggml_set_f32(targets, -1.0f);
10621063
ggml_set_i32_1d(tokens_input, 0, 0);
10631064
for (int i=1; i<n_tokens+1; ++i) {
10641065
float x = example_id + i * 3.14159f * 2.0f * 1.0f / n_tokens;

0 commit comments

Comments
 (0)