Pass provider inputs in raw payloads

Author: Wauplin

packages/inference/src/snippets/python.ts

@@ -17,6 +17,7 @@ interface TemplateParams {
 	baseUrl?: string;
 	fullUrl?: string;
 	inputs?: object;
+	providerInputs?: object;
 	model?: ModelDataMinimal;
 	provider?: InferenceProvider;
 	providerModelId?: string;
@@ -115,6 +116,18 @@ const snippetGenerator = (templateName: string, inputPreparationFn?: InputPrepar
 			{ chatCompletion: templateName.includes("conversational"), task: model.pipeline_tag as InferenceTask }
 		);
 
+		/// Parse request.info.body if not a binary.
+		/// This is the body sent to the provider. Important for snippets with raw payload (e.g curl, requests, etc.)
+		let providerInputs = inputs;
+		const bodyAsObj = request.info.body;
+		if (typeof bodyAsObj === "string") {
+			try {
+				providerInputs = JSON.parse(bodyAsObj);
+			} catch (e) {
+				console.error("Failed to parse body as JSON", e);
+			}
+		}
+
 		/// Prepare template injection data
 		const params: TemplateParams = {
 			accessToken,
@@ -126,6 +139,11 @@ const snippetGenerator = (templateName: string, inputPreparationFn?: InputPrepar
 				asJsonString: formatBody(inputs, "json"),
 				asPythonString: indentString(formatBody(inputs, "python"), 4),
 			},
+			providerInputs: {
+				asObj: providerInputs,
+				asJsonString: formatBody(providerInputs, "json"),
+				asPythonString: indentString(formatBody(providerInputs, "python"), 4),
+			},
 			model,
 			provider,
 			providerModelId: providerModelId ?? model.id,

packages/inference/src/snippets/templates/python/requests/automaticSpeechRecognition.jinja

@@ -4,4 +4,4 @@ def query(filename):
     response = requests.post(API_URL, headers=headers, data=data)
     return response.json()
 
-output = query({{ inputs.asObj.inputs }})
+output = query({{ providerInputs.asObj.inputs }})

packages/inference/src/snippets/templates/python/requests/basic.jinja

@@ -3,5 +3,5 @@ def query(payload):
     return response.json()
 
 output = query({
-    "inputs": {{ inputs.asObj.inputs }},
+    "inputs": {{ providerInputs.asObj.inputs }},
 }) 

packages/inference/src/snippets/templates/python/requests/basicFile.jinja

@@ -4,4 +4,4 @@ def query(filename):
     response = requests.post(API_URL, headers=headers, data=data)
     return response.json()
 
-output = query({{ inputs.asObj.inputs }})
+output = query({{ providerInputs.asObj.inputs }})

packages/inference/src/snippets/templates/python/requests/conversational.jinja

@@ -3,8 +3,7 @@ def query(payload):
     return response.json()
 
 response = query({
-    "model": "{{ providerModelId }}",
-{{ inputs.asJsonString }}
+{{ providerInputs.asJsonString }}
 })
 
 print(response["choices"][0]["message"])

packages/inference/src/snippets/templates/python/requests/conversationalStream.jinja

@@ -8,8 +8,7 @@ def query(payload):
         yield json.loads(line.decode("utf-8").lstrip("data:").rstrip("/n"))
 
 chunks = query({
-    "model": "{{ providerModelId }}",
-{{ inputs.asJsonString }},
+{{ providerInputs.asJsonString }},
     "stream": True,
 })
 

packages/inference/src/snippets/templates/python/requests/documentQuestionAnswering.jinja

@@ -6,5 +6,5 @@ def query(payload):
     return response.json()
 
 output = query({
-{{ inputs.asJsonString }},
+{{ providerInputs.asJsonString }},
 }) 

packages/inference/src/snippets/templates/python/requests/imageToImage.jinja

@@ -6,7 +6,7 @@ def query(payload):
     return response.content
 
 image_bytes = query({
-{{ inputs.asJsonString }}
+{{ providerInputs.asJsonString }}
 })
 
 # You can access the image with PIL.Image for example

packages/inference/src/snippets/templates/python/requests/tabular.jinja

@@ -4,6 +4,6 @@ def query(payload):
 
 response = query({
     "inputs": {
-        "data": {{ inputs.asObj.inputs }}
+        "data": {{ providerInputs.asObj.inputs }}
     },
 }) 

packages/inference/src/snippets/templates/python/requests/textToAudio.jinja

@@ -4,7 +4,7 @@ def query(payload):
     return response.content
 
 audio_bytes = query({
-    "inputs": {{ inputs.asObj.inputs }},
+    "inputs": {{ providerInputs.asObj.inputs }},
 })
 # You can access the audio with IPython.display for example
 from IPython.display import Audio
@@ -15,7 +15,7 @@ def query(payload):
     return response.json()
 
 audio, sampling_rate = query({
-    "inputs": {{ inputs.asObj.inputs }},
+    "inputs": {{ providerInputs.asObj.inputs }},
 })
 # You can access the audio with IPython.display for example
 from IPython.display import Audio

packages/inference/src/snippets/templates/python/requests/textToImage.jinja

@@ -4,7 +4,7 @@ def query(payload):
     return response.content
 
 image_bytes = query({
-    "inputs": {{ inputs.asObj.inputs }},
+    "inputs": {{ providerInputs.asObj.inputs }},
 })
 
 # You can access the image with PIL.Image for example

packages/inference/src/snippets/templates/python/requests/zeroShotClassification.jinja

@@ -3,6 +3,6 @@ def query(payload):
     return response.json()
 
 output = query({
-    "inputs": {{ inputs.asObj.inputs }},
+    "inputs": {{ providerInputs.asObj.inputs }},
     "parameters": {"candidate_labels": ["refund", "legal", "faq"]},
 }) 

packages/inference/src/snippets/templates/python/requests/zeroShotImageClassification.jinja

@@ -9,6 +9,6 @@ def query(data):
     return response.json()
 
 output = query({
-    "image_path": {{ inputs.asObj.inputs }},
+    "image_path": {{ providerInputs.asObj.inputs }},
     "parameters": {"candidate_labels": ["cat", "dog", "llama"]},
 }) 

packages/tasks-gen/snippets-fixtures/conversational-llm-non-stream/1.requests.hf-inference.py

@@ -8,14 +8,14 @@ def query(payload):
     return response.json()
 
 response = query({
-    "model": "meta-llama/Llama-3.1-8B-Instruct",
     "messages": [
         {
             "role": "user",
             "content": "What is the capital of France?"
         }
     ],
-    "max_tokens": 500
+    "max_tokens": 500,
+    "model": "meta-llama/Llama-3.1-8B-Instruct"
 })
 
 print(response["choices"][0]["message"])

packages/tasks-gen/snippets-fixtures/conversational-llm-non-stream/1.requests.together.py

@@ -8,14 +8,14 @@ def query(payload):
     return response.json()
 
 response = query({
-    "model": "<together alias for meta-llama/Llama-3.1-8B-Instruct>",
     "messages": [
         {
             "role": "user",
             "content": "What is the capital of France?"
         }
     ],
-    "max_tokens": 500
+    "max_tokens": 500,
+    "model": "<together alias for meta-llama/Llama-3.1-8B-Instruct>"
 })
 
 print(response["choices"][0]["message"])

packages/tasks-gen/snippets-fixtures/conversational-llm-stream/1.requests.hf-inference.py

@@ -14,14 +14,14 @@ def query(payload):
         yield json.loads(line.decode("utf-8").lstrip("data:").rstrip("/n"))
 
 chunks = query({
-    "model": "meta-llama/Llama-3.1-8B-Instruct",
     "messages": [
         {
             "role": "user",
             "content": "What is the capital of France?"
         }
     ],
     "max_tokens": 500,
+    "model": "meta-llama/Llama-3.1-8B-Instruct",
     "stream": True,
 })
 

packages/tasks-gen/snippets-fixtures/conversational-llm-stream/1.requests.together.py

@@ -14,14 +14,14 @@ def query(payload):
         yield json.loads(line.decode("utf-8").lstrip("data:").rstrip("/n"))
 
 chunks = query({
-    "model": "<together alias for meta-llama/Llama-3.1-8B-Instruct>",
     "messages": [
         {
             "role": "user",
             "content": "What is the capital of France?"
         }
     ],
     "max_tokens": 500,
+    "model": "<together alias for meta-llama/Llama-3.1-8B-Instruct>",
     "stream": True,
 })
 

packages/tasks-gen/snippets-fixtures/conversational-vlm-non-stream/1.requests.fireworks-ai.py

@@ -8,7 +8,6 @@ def query(payload):
     return response.json()
 
 response = query({
-    "model": "<fireworks-ai alias for meta-llama/Llama-3.2-11B-Vision-Instruct>",
     "messages": [
         {
             "role": "user",
@@ -26,7 +25,8 @@ def query(payload):
             ]
         }
     ],
-    "max_tokens": 500
+    "max_tokens": 500,
+    "model": "<fireworks-ai alias for meta-llama/Llama-3.2-11B-Vision-Instruct>"
 })
 
 print(response["choices"][0]["message"])

packages/tasks-gen/snippets-fixtures/conversational-vlm-non-stream/1.requests.hf-inference.py

@@ -8,7 +8,6 @@ def query(payload):
     return response.json()
 
 response = query({
-    "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
     "messages": [
         {
             "role": "user",
@@ -26,7 +25,8 @@ def query(payload):
             ]
         }
     ],
-    "max_tokens": 500
+    "max_tokens": 500,
+    "model": "meta-llama/Llama-3.2-11B-Vision-Instruct"
 })
 
 print(response["choices"][0]["message"])

packages/tasks-gen/snippets-fixtures/conversational-vlm-stream/1.requests.fireworks-ai.py

@@ -14,7 +14,6 @@ def query(payload):
         yield json.loads(line.decode("utf-8").lstrip("data:").rstrip("/n"))
 
 chunks = query({
-    "model": "<fireworks-ai alias for meta-llama/Llama-3.2-11B-Vision-Instruct>",
     "messages": [
         {
             "role": "user",
@@ -33,6 +32,7 @@ def query(payload):
         }
     ],
     "max_tokens": 500,
+    "model": "<fireworks-ai alias for meta-llama/Llama-3.2-11B-Vision-Instruct>",
     "stream": True,
 })
 

packages/tasks-gen/snippets-fixtures/conversational-vlm-stream/1.requests.hf-inference.py

@@ -14,7 +14,6 @@ def query(payload):
         yield json.loads(line.decode("utf-8").lstrip("data:").rstrip("/n"))
 
 chunks = query({
-    "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
     "messages": [
         {
             "role": "user",
@@ -33,6 +32,7 @@ def query(payload):
         }
     ],
     "max_tokens": 500,
+    "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
     "stream": True,
 })