Add seed, temperature, max_tokens and system_fingerprint paramters to request/response

Author: vmpuri

api/api.py

@@ -105,20 +105,20 @@ class CompletionRequest:
     logit_bias: Optional[Dict[str, float]] = None  # unimplemented
     logprobs: Optional[bool] = None  # unimplemented
     top_logprobs: Optional[int] = None  # unimplemented
-    max_tokens: Optional[int] = None  # unimplemented
+    max_tokens: Optional[int] = None
     n: int = 1
     presence_penalty: float = 0  # unimplemented
     response_format: Optional[ResponseFormat] = None  # unimplemented
-    seed: Optional[int] = None  # unimplemented
+    seed: Optional[int] = None
     service_tier: Optional[str] = None  # unimplemented
     stop: Optional[List[str]] = None  # unimplemented
     stream: bool = False
     stream_options: Optional[StreamOptions] = None  # unimplemented
-    temperature: Optional[float] = 1.0  # unimplemented
+    temperature: Optional[float] = 1.0
     top_p: Optional[float] = 1.0  # unimplemented
-    tools: Optional[List[Any]] = None  # unimplemented
-    tool_choice: Optional[Union[str, Any]] = None  # unimplemented
-    parallel_tool_calls: Optional[bool] = None  # unimplemented
+    tools: Optional[List[Any]] = None  # unimplemented - Assistant features
+    tool_choice: Optional[Union[str, Any]] = None  # unimplemented - Assistant features
+    parallel_tool_calls: Optional[bool] = None  # unimplemented - Assistant features
     user: Optional[str] = None  # unimplemented
 
 
@@ -229,9 +229,8 @@ def __init__(self, *args, **kwargs):
             else self.model.config.max_seq_length
         )
         # The System fingerprint is a unique identifier for the model and its configuration.
-        # Currently, this is not implemented in a
         self.system_fingerprint = (
-            self.builder_args.device + type(self.builder_args.precision).__name__
+            self.builder_args.device + "_" + str(self.builder_args.precision)
         )
 
     def chunked_completion(self, completion_request: CompletionRequest):
@@ -270,7 +269,11 @@ def chunked_completion(self, completion_request: CompletionRequest):
         )
         generator_args = GeneratorArgs(
             completion_request.messages[-1].get("content"),
+            max_new_tokens=(
+                completion_request.max_tokens if completion_request.max_tokens else 16
+            ),
             encoded_prompt=encoded,
+            temperature=completion_request.temperature,
             chat_mode=False,
         )
 

generate.py

@@ -71,7 +71,7 @@ class GeneratorArgs:
     num_samples: int = 1
     max_new_tokens: int = 200
     top_k: int = 200
-    temperature: int = 0  # deterministic argmax
+    temperature: float = 0  # deterministic argmax if 0.0
     compile: bool = False
     compile_prefill: bool = False
     speculate_k: int = 5
@@ -105,9 +105,7 @@ def validate_build(
     def from_args(cls, args):
         dso_path = getattr(args, "dso_path", None)
         pte_path = getattr(args, "pte_path", None)
-        sequential_prefill = (
-            args.sequential_prefill or bool(dso_path) or bool(pte_path)
-        )
+        sequential_prefill = args.sequential_prefill or bool(dso_path) or bool(pte_path)
 
         return cls(
             prompt=getattr(args, "prompt", ""),

server.py

@@ -9,6 +9,8 @@
 from dataclasses import asdict
 from typing import Dict, List, Union
 
+import torch
+
 from api.api import CompletionRequest, OpenAiApiGenerator
 from api.models import get_model_info_list, retrieve_model_info
 
@@ -50,6 +52,8 @@ def chat_endpoint():
         """
 
         print(" === Completion Request ===")
+        if seed := request.args.get("seed"):
+            torch.manual_seed(int(seed))
 
         # Parse the request in to a CompletionRequest object
         data = request.get_json()