Better llama.cpp interoperability

Has some too many newline issues so WIP

(Update) Fixed too many newlines, now onto args.

Still needs shipping work so you could do "python -m llama_cpp.examples." etc.

Author: Mug

examples/common.py

@@ -0,0 +1,135 @@
+import os
+import argparse
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+# Based on https://github.com/ggerganov/llama.cpp/blob/master/examples/common.cpp
+
+
+@dataclass
+class GptParams:
+    seed: int = -1
+    n_threads: int = min(4, os.cpu_count() or 1)
+    n_predict: int = 128
+    repeat_last_n: int = 64
+    n_parts: int = -1
+    n_ctx: int = 512
+    n_batch: int = 8
+    n_keep: int = 0
+
+    top_k: int = 40
+    top_p: float = 0.95
+    temp: float = 0.80
+    repeat_penalty: float = 1.10
+
+    model: str = "./models/llama-7B/ggml-model.bin"
+    prompt: str = ""
+    input_prefix: str = " "
+    fix_prefix: str = ""
+    output_postfix: str = ""
+    input_echo: bool = True,
+
+    antiprompt: List[str] = field(default_factory=list)
+
+    memory_f16: bool = True
+    random_prompt: bool = False
+    use_color: bool = False
+    interactive: bool = False
+
+    embedding: bool = False
+    interactive_start: bool = False
+
+    instruct: bool = False
+    ignore_eos: bool = False
+    perplexity: bool = False
+    use_mlock: bool = False
+    mem_test: bool = False
+    verbose_prompt: bool = False
+
+    # Default instructions for Alpaca
+    # switch to "Human" and "Assistant" for Vicuna.
+    instruct_inp_prefix: str="\n\n### Instruction:\n\n",
+    instruct_inp_suffix: str="\n\n### Response:\n\n",
+
+
+def gpt_params_parse(argv = None, params: Optional[GptParams] = None):
+    if params is None:
+        params = GptParams()
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-h", "--help", action="store_true", help="show this help message and exit")
+    parser.add_argument("-s", "--seed", type=int, default=-1, help="",dest="seed")
+    parser.add_argument("-t", "--threads", type=int, default=1, help="",dest="n_threads")
+    parser.add_argument("-p", "--prompt", type=str, default="", help="",dest="prompt")
+    parser.add_argument("-f", "--file", type=str, default=None, help="")
+    parser.add_argument("-c", "--ctx_size", type=int, default=512, help="",dest="n_ctx")
+    parser.add_argument("--memory_f32", action="store_false", help="",dest="memory_f16")
+    parser.add_argument("--top_p", type=float, default=0.9, help="",dest="top_p")
+    parser.add_argument("--temp", type=float, default=1.0, help="",dest="temp")
+    parser.add_argument("--repeat_last_n", type=int, default=64, help="",dest="repeat_last_n")
+    parser.add_argument("--repeat_penalty", type=float, default=1.0, help="",dest="repeat_penalty")
+    parser.add_argument("-b", "--batch_size", type=int, default=8, help="",dest="n_batch")
+    parser.add_argument("--keep", type=int, default=0, help="",dest="n_keep")
+    parser.add_argument("-m", "--model", type=str, help="",dest="model")
+    parser.add_argument(
+        "-i", "--interactive", action="store_true", help="run in interactive mode", dest="interactive"
+    )
+    parser.add_argument("--embedding", action="store_true", help="", dest="embedding")
+    parser.add_argument("--interactive-start", action="store_true", help="", dest="interactive_start")
+    parser.add_argument(
+        "--interactive-first",
+        action="store_true",
+        help="run in interactive mode and wait for input right away",
+        dest="interactive"
+    )
+    parser.add_argument(
+        "-ins",
+        "--instruct",
+        action="store_true",
+        help="run in instruction mode (use with Alpaca or Vicuna models)",
+        dest="instruct"
+    )
+    parser.add_argument(
+        "--color",
+        action="store_true",
+        help="colorise output to distinguish prompt and user input from generations",
+        dest="use_color"
+    )
+    parser.add_argument("--mlock", action="store_true",dest="use_mlock")
+    parser.add_argument("--mtest", action="store_true",dest="mem_test")
+    parser.add_argument(
+        "-r",
+        "--reverse-prompt",
+        type=str,
+        action='append',
+        help="run in interactive mode and poll user input upon seeing PROMPT (can be\nspecified more than once for multiple prompts).",
+        dest="antiprompt"
+    )
+    parser.add_argument("--perplexity", action="store_true", help="", dest="perplexity")
+    parser.add_argument("--ignore-eos", action="store_true", help="", dest="ignore_eos")
+    parser.add_argument("--n_parts", type=int, default=-1, help="", dest="n_parts")
+    parser.add_argument("--random-prompt", action="store_true", help="", dest="random_prompt")
+    parser.add_argument("--in-prefix", type=str, default=" ", help="", dest="input_prefix")
+    parser.add_argument("--fix-prefix", type=str, default=" ", help="", dest="fix_prefix")
+    parser.add_argument("--out-postfix", type=str, default="", help="", dest="output_postfix")
+    parser.add_argument("--input-noecho", action="store_false", help="", dest="input_echo")
+    args = parser.parse_args(argv)
+    return args
+
+def gpt_random_prompt(rng):
+    return [
+        "So",
+        "Once upon a time",
+        "When",
+        "The",
+        "After",
+        "If",
+        "import",
+        "He",
+        "She",
+        "They",
+    ][rng % 10]
+
+if __name__ == "__main__":
+    print(GptParams(gpt_params_parse()))