|
| 1 | +// Defines sigaction on msys: |
| 2 | +#ifndef _GNU_SOURCE |
| 3 | +#define _GNU_SOURCE |
| 4 | +#endif |
| 5 | + |
| 6 | +#include "embd-input.h" |
| 7 | + |
| 8 | +#include <cassert> |
| 9 | +#include <cinttypes> |
| 10 | +#include <cmath> |
| 11 | +#include <cstdio> |
| 12 | +#include <cstring> |
| 13 | +#include <ctime> |
| 14 | +#include <fstream> |
| 15 | +#include <iostream> |
| 16 | +#include <string> |
| 17 | +#include <vector> |
| 18 | + |
| 19 | +static llama_context ** g_ctx; |
| 20 | + |
| 21 | +extern "C" { |
| 22 | + |
| 23 | +struct MyModel* create_mymodel(int argc, char ** argv) { |
| 24 | + gpt_params params; |
| 25 | + |
| 26 | + if (gpt_params_parse(argc, argv, params) == false) { |
| 27 | + return nullptr; |
| 28 | + } |
| 29 | + |
| 30 | + fprintf(stderr, "%s: build = %d (%s)\n", __func__, BUILD_NUMBER, BUILD_COMMIT); |
| 31 | + |
| 32 | + if (params.seed < 0) { |
| 33 | + params.seed = time(NULL); |
| 34 | + } |
| 35 | + fprintf(stderr, "%s: seed = %d\n", __func__, params.seed); |
| 36 | + |
| 37 | + llama_init_backend(params.numa); |
| 38 | + |
| 39 | + llama_model * model; |
| 40 | + llama_context * ctx; |
| 41 | + |
| 42 | + g_ctx = &ctx; |
| 43 | + |
| 44 | + // load the model and apply lora adapter, if any |
| 45 | + std::tie(model, ctx) = llama_init_from_gpt_params(params); |
| 46 | + if (model == NULL) { |
| 47 | + fprintf(stderr, "%s: error: unable to load model\n", __func__); |
| 48 | + return nullptr; |
| 49 | + } |
| 50 | + |
| 51 | + // print system information |
| 52 | + { |
| 53 | + fprintf(stderr, "\n"); |
| 54 | + fprintf(stderr, "system_info: n_threads = %d / %d | %s\n", |
| 55 | + params.n_threads, std::thread::hardware_concurrency(), llama_print_system_info()); |
| 56 | + } |
| 57 | + struct MyModel * ret = new MyModel(); |
| 58 | + ret->ctx = ctx; |
| 59 | + ret->params = params; |
| 60 | + ret->n_past = 0; |
| 61 | + // printf("ctx: %d\n", ret->ctx); |
| 62 | + return ret; |
| 63 | +} |
| 64 | + |
| 65 | +void free_mymodel(struct MyModel * mymodel) { |
| 66 | + llama_context * ctx = mymodel->ctx; |
| 67 | + llama_print_timings(ctx); |
| 68 | + llama_free(ctx); |
| 69 | + delete mymodel; |
| 70 | +} |
| 71 | + |
| 72 | + |
| 73 | +bool eval_float(void * model, float * input, int N){ |
| 74 | + MyModel * mymodel = (MyModel*)model; |
| 75 | + llama_context * ctx = mymodel->ctx; |
| 76 | + gpt_params params = mymodel->params; |
| 77 | + int n_emb = llama_n_embd(ctx); |
| 78 | + int n_past = mymodel->n_past; |
| 79 | + int n_batch = N; // params.n_batch; |
| 80 | + |
| 81 | + for (int i = 0; i < (int) N; i += n_batch) { |
| 82 | + int n_eval = (int) N - i; |
| 83 | + if (n_eval > n_batch) { |
| 84 | + n_eval = n_batch; |
| 85 | + } |
| 86 | + if (llama_eval_embd(ctx, (input+i*n_emb), n_eval, n_past, params.n_threads)) { |
| 87 | + fprintf(stderr, "%s : failed to eval\n", __func__); |
| 88 | + return false; |
| 89 | + } |
| 90 | + n_past += n_eval; |
| 91 | + } |
| 92 | + mymodel->n_past = n_past; |
| 93 | + return true; |
| 94 | +} |
| 95 | + |
| 96 | +bool eval_tokens(void * model, std::vector<llama_token> tokens) { |
| 97 | + MyModel * mymodel = (MyModel* )model; |
| 98 | + llama_context * ctx; |
| 99 | + ctx = mymodel->ctx; |
| 100 | + gpt_params params = mymodel->params; |
| 101 | + int n_past = mymodel->n_past; |
| 102 | + for (int i = 0; i < (int) tokens.size(); i += params.n_batch) { |
| 103 | + int n_eval = (int) tokens.size() - i; |
| 104 | + if (n_eval > params.n_batch) { |
| 105 | + n_eval = params.n_batch; |
| 106 | + } |
| 107 | + if (llama_eval(ctx, &tokens[i], n_eval, n_past, params.n_threads)) { |
| 108 | + fprintf(stderr, "%s : failed to eval\n", __func__); |
| 109 | + return false; |
| 110 | + } |
| 111 | + n_past += n_eval; |
| 112 | + } |
| 113 | + mymodel->n_past = n_past; |
| 114 | + return true; |
| 115 | +} |
| 116 | + |
| 117 | +bool eval_id(struct MyModel* mymodel, int id) { |
| 118 | + std::vector<llama_token> tokens; |
| 119 | + tokens.push_back(id); |
| 120 | + return eval_tokens(mymodel, tokens); |
| 121 | +} |
| 122 | + |
| 123 | +bool eval_string(struct MyModel * mymodel,const char* str){ |
| 124 | + llama_context * ctx = mymodel->ctx; |
| 125 | + std::string str2 = str; |
| 126 | + std::vector<llama_token> embd_inp = ::llama_tokenize(ctx, str2, true); |
| 127 | + eval_tokens(mymodel, embd_inp); |
| 128 | + return true; |
| 129 | +} |
| 130 | + |
| 131 | +llama_token sampling_id(struct MyModel* mymodel) { |
| 132 | + llama_context* ctx = mymodel->ctx; |
| 133 | + gpt_params params = mymodel->params; |
| 134 | + // int n_ctx = llama_n_ctx(ctx); |
| 135 | + |
| 136 | + // out of user input, sample next token |
| 137 | + const float temp = params.temp; |
| 138 | + const int32_t top_k = params.top_k <= 0 ? llama_n_vocab(ctx) : params.top_k; |
| 139 | + const float top_p = params.top_p; |
| 140 | + const float tfs_z = params.tfs_z; |
| 141 | + const float typical_p = params.typical_p; |
| 142 | + // const int32_t repeat_last_n = params.repeat_last_n < 0 ? n_ctx : params.repeat_last_n; |
| 143 | + // const float repeat_penalty = params.repeat_penalty; |
| 144 | + // const float alpha_presence = params.presence_penalty; |
| 145 | + // const float alpha_frequency = params.frequency_penalty; |
| 146 | + const int mirostat = params.mirostat; |
| 147 | + const float mirostat_tau = params.mirostat_tau; |
| 148 | + const float mirostat_eta = params.mirostat_eta; |
| 149 | + // const bool penalize_nl = params.penalize_nl; |
| 150 | + |
| 151 | + llama_token id = 0; |
| 152 | + { |
| 153 | + auto logits = llama_get_logits(ctx); |
| 154 | + auto n_vocab = llama_n_vocab(ctx); |
| 155 | + |
| 156 | + // Apply params.logit_bias map |
| 157 | + for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) { |
| 158 | + logits[it->first] += it->second; |
| 159 | + } |
| 160 | + |
| 161 | + std::vector<llama_token_data> candidates; |
| 162 | + candidates.reserve(n_vocab); |
| 163 | + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { |
| 164 | + candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f}); |
| 165 | + } |
| 166 | + |
| 167 | + llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; |
| 168 | + |
| 169 | + // TODO: Apply penalties |
| 170 | + // float nl_logit = logits[llama_token_nl()]; |
| 171 | + // auto last_n_repeat = std::min(std::min((int)last_n_tokens.size(), repeat_last_n), n_ctx); |
| 172 | + // llama_sample_repetition_penalty(ctx, &candidates_p, |
| 173 | + // last_n_tokens.data() + last_n_tokens.size() - last_n_repeat, |
| 174 | + // last_n_repeat, repeat_penalty); |
| 175 | + // llama_sample_frequency_and_presence_penalties(ctx, &candidates_p, |
| 176 | + // last_n_tokens.data() + last_n_tokens.size() - last_n_repeat, |
| 177 | + // last_n_repeat, alpha_frequency, alpha_presence); |
| 178 | + // if (!penalize_nl) { |
| 179 | + // logits[llama_token_nl()] = nl_logit; |
| 180 | + // } |
| 181 | + |
| 182 | + if (temp <= 0) { |
| 183 | + // Greedy sampling |
| 184 | + id = llama_sample_token_greedy(ctx, &candidates_p); |
| 185 | + } else { |
| 186 | + if (mirostat == 1) { |
| 187 | + static float mirostat_mu = 2.0f * mirostat_tau; |
| 188 | + const int mirostat_m = 100; |
| 189 | + llama_sample_temperature(ctx, &candidates_p, temp); |
| 190 | + id = llama_sample_token_mirostat(ctx, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu); |
| 191 | + } else if (mirostat == 2) { |
| 192 | + static float mirostat_mu = 2.0f * mirostat_tau; |
| 193 | + llama_sample_temperature(ctx, &candidates_p, temp); |
| 194 | + id = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu); |
| 195 | + } else { |
| 196 | + // Temperature sampling |
| 197 | + llama_sample_top_k(ctx, &candidates_p, top_k, 1); |
| 198 | + llama_sample_tail_free(ctx, &candidates_p, tfs_z, 1); |
| 199 | + llama_sample_typical(ctx, &candidates_p, typical_p, 1); |
| 200 | + llama_sample_top_p(ctx, &candidates_p, top_p, 1); |
| 201 | + llama_sample_temperature(ctx, &candidates_p, temp); |
| 202 | + id = llama_sample_token(ctx, &candidates_p); |
| 203 | + } |
| 204 | + } |
| 205 | + } |
| 206 | + |
| 207 | + return id; |
| 208 | +} |
| 209 | + |
| 210 | +const char * sampling(struct MyModel * mymodel) { |
| 211 | + llama_context * ctx = mymodel->ctx; |
| 212 | + int id = sampling_id(mymodel); |
| 213 | + std::string ret; |
| 214 | + if (id == llama_token_eos()) ret = "</s>"; |
| 215 | + else ret = llama_token_to_str(ctx, id); |
| 216 | + eval_id(mymodel, id); |
| 217 | + return ret.c_str(); |
| 218 | +} |
| 219 | + |
| 220 | +} |
0 commit comments