Add basic cpu setup

Author: ltoniazzi

BRANCH_SETUP.md

@@ -0,0 +1,48 @@
+# Setup this branch
+
+## Create a lora adpter bin file
+
+0. `mkdir models/open-llama` and download [Open-llama  (all files)](https://huggingface.co/openlm-research/open_llama_3b_v2/tree/main) in the folder `./models/open-llama`
+
+2. `mkdir data && touch data/hot-lora.txt` and write a couple of words in it.
+
+3. Run:
+    ```bash
+    # Convert base model to gguf
+    python3 convert-hf-to-gguf.py models/open-llama/
+    # Quantize base model
+    ./quantize ./models/open-llama/ggml-model-f16.gguf ./models/open-llama/ggml-model-q8_0.gguf Q8_0
+    # Obtain Lora adapter
+    ./finetune  --model-base models/open-llama/ggml-model-q8_0.gguf \
+    --checkpoint-in models/open-llama/chk-lora-ggml-model-q8_0-hot-lora-LATEST.gguf \
+    --checkpoint-out models/open-llama/chk-lora-ggml-model-q8_0-hot-lora-ITERATION.gguf \
+    --lora-out models/open-llama/lora-ggml-model-q8_0-hot-lora-ITERATION.bin \
+    --train-data "data/hot-lora.txt" \
+    --save-every 1 \
+    --threads 1 \
+    --adam-iter 1 \
+    --batch 1 \
+    --ctx 16 \
+    --use-checkpointing
+    ```
+
+## Run main with adapter
+
+Run main with base model and lora adapter to hot-swap
+```bash
+./main ./models/open-llama/ggml-model-f16.gguf \
+--hot-lora models/open-llama/lora-ggml-model-q8_0-hot-lora-ITERATION.bin \
+-ngl 0 \
+-n 128
+```
+
+With `ngl > 0` the code breaks. Probably because the Lora tensors try to interact with the base tensors (`lora_mul_mat`), but they are not moved to the buffer of the base tensors.
+
+# Logic
+
+
+
+# Current status
+
+- Only ony Lora adapter can be passed. 
+- GPU not supported

common/common.cpp

@@ -789,6 +789,14 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
         params.model = argv[i];
         return true;
     }
+    if (arg == "-hl" || arg == "--hot-lora") {
+        if (++i >= argc) {
+            invalid_param = true;
+            return true;
+        }
+        params.hot_lora = argv[i];
+        return true;
+    }
     if (arg == "-md" || arg == "--model-draft") {
         if (++i >= argc) {
             invalid_param = true;

common/common.h

@@ -100,6 +100,7 @@ struct gpt_params {
 
     std::string model                = ""; // model path
     std::string model_draft          = ""; // draft model for speculative decoding
+    std::string hot_lora             = ""; // lora model path for hot swapping
     std::string model_alias          = "unknown"; // model alias
     std::string model_url            = ""; // model url to download
     std::string hf_repo              = ""; // HF repo

data/hot-lora.txt

@@ -0,0 +1,2 @@
+
+ how are you?

ggml.c

@@ -4313,6 +4313,52 @@ struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * nam
     return NULL;
 }
 
+//////// LORA
+
+struct lora_tensor_pair* build_lora_weights_map(struct ggml_context* ctx) {
+    struct lora_tensor_pair* pair = malloc(sizeof(struct lora_tensor_pair));
+    if (!pair) return NULL;
+    pair->pairs = NULL;
+    pair->count = 0;
+    pair->capacity = 0;
+
+    struct ggml_object * obj = ctx->objects_begin;
+    char * const mem_buffer = ctx->mem_buffer;
+
+    while (obj != NULL) {
+        if (obj->type == GGML_OBJECT_TYPE_TENSOR) {
+            struct ggml_tensor * tensor = (struct ggml_tensor *)(mem_buffer + obj->offs);
+            char * tensor_name = tensor->name;
+
+            if (strlen(tensor_name) > 6 && (strcmp(tensor_name + strlen(tensor_name) - 6, ".loraA") == 0 ||
+                                            strcmp(tensor_name + strlen(tensor_name) - 6, ".loraB") == 0)) {
+                if (pair->count == pair->capacity) {
+                    pair->capacity = pair->capacity > 0 ? pair->capacity * 2 : 4;
+                    pair->pairs = realloc(pair->pairs, pair->capacity * sizeof(struct lora_tensor_info));
+                }
+
+                pair->pairs[pair->count].name = strdup(tensor_name);
+                pair->pairs[pair->count].tensor = tensor;
+                pair->count++;
+            }
+        }
+        obj = obj->next;
+    }
+
+    return pair;
+}
+
+void free_lora_tensor_pair(struct lora_tensor_pair* pair) {
+    if (!pair) return;
+    for (int i = 0; i < pair->count; i++) {
+        free(pair->pairs[i].name);
+    }
+    free(pair->pairs);
+    free(pair);
+}
+
+//////// LORA
+
 ////////////////////////////////////////////////////////////////////////////////
 
 // ggml_dup

ggml.h

@@ -835,6 +835,25 @@ extern "C" {
     GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);
     GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);
 
+    struct lora_tensor_info {
+        char* name;
+        struct ggml_tensor* tensor;
+        };
+
+        struct lora_tensor_pair {
+        struct lora_tensor_info* pairs; // Dynamic array of tensor pairs
+        int count;
+        int capacity;
+        };
+
+        // Function to build tensor pairs
+        struct lora_tensor_pair* build_lora_weights_map(struct ggml_context* ctx);
+
+        // Cleanup function for lora_tensor_pair
+        void free_lora_tensor_pair(struct lora_tensor_pair* pair);
+
+
+
     GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
     GGML_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);
     GGML_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);