Skip to content

Commit fc57b44

Browse files
mishig25mishig25
authored
[Inference snippets] VLM hf_hub, oai snippets (#985)
Thanks to #976, now we can show `hf_hub`, `oai` snippets for VLMs ("conversational image-text-to-text" models).
1 parent 9cb9c0d commit fc57b44

File tree

3 files changed

+58
-98
lines changed

3 files changed

+58
-98
lines changed

‎packages/tasks/src/snippets/curl.ts

Lines changed: 19 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,24 @@ export const snippetTextGeneration = (
2626
if (model.tags.includes("conversational")) {
2727
// Conversational model detected, so we display a code snippet that features the Messages API
2828
const streaming = opts?.streaming ?? true;
29-
const messages: ChatCompletionInputMessage[] = opts?.messages ?? [
30-
{ role: "user", content: "What is the capital of France?" },
31-
];
29+
const exampleMessages: ChatCompletionInputMessage[] =
30+
model.pipeline_tag === "text-generation"
31+
? [{ role: "user", content: "What is the capital of France?" }]
32+
: [
33+
{
34+
role: "user",
35+
content: [
36+
{
37+
type: "image_url",
38+
image_url: {
39+
url: "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg",
40+
},
41+
},
42+
{ type: "text", text: "Describe this image in one sentence." },
43+
],
44+
},
45+
];
46+
const messages = opts?.messages ?? exampleMessages;
3247

3348
const config = {
3449
...(opts?.temperature ? { temperature: opts.temperature } : undefined),
@@ -63,34 +78,6 @@ export const snippetTextGeneration = (
6378
}
6479
};
6580

66-
export const snippetImageTextToTextGeneration = (model: ModelDataMinimal, accessToken: string): InferenceSnippet => {
67-
if (model.tags.includes("conversational")) {
68-
// Conversational model detected, so we display a code snippet that features the Messages API
69-
return {
70-
content: `curl 'https://api-inference.huggingface.co/models/${model.id}/v1/chat/completions' \\
71-
-H "Authorization: Bearer ${accessToken || `{API_TOKEN}`}" \\
72-
-H 'Content-Type: application/json' \\
73-
-d '{
74-
"model": "${model.id}",
75-
"messages": [
76-
{
77-
"role": "user",
78-
"content": [
79-
{"type": "image_url", "image_url": {"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"}},
80-
{"type": "text", "text": "Describe this image in one sentence."}
81-
]
82-
}
83-
],
84-
"max_tokens": 500,
85-
"stream": false
86-
}'
87-
`,
88-
};
89-
} else {
90-
return snippetBasic(model, accessToken);
91-
}
92-
};
93-
9481
export const snippetZeroShotClassification = (model: ModelDataMinimal, accessToken: string): InferenceSnippet => ({
9582
content: `curl https://api-inference.huggingface.co/models/${model.id} \\
9683
-X POST \\
@@ -122,7 +109,7 @@ export const curlSnippets: Partial<
122109
summarization: snippetBasic,
123110
"feature-extraction": snippetBasic,
124111
"text-generation": snippetTextGeneration,
125-
"image-text-to-text": snippetImageTextToTextGeneration,
112+
"image-text-to-text": snippetTextGeneration,
126113
"text2text-generation": snippetBasic,
127114
"fill-mask": snippetBasic,
128115
"sentence-similarity": snippetBasic,

‎packages/tasks/src/snippets/js.ts

Lines changed: 19 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,24 @@ export const snippetTextGeneration = (
4040
if (model.tags.includes("conversational")) {
4141
// Conversational model detected, so we display a code snippet that features the Messages API
4242
const streaming = opts?.streaming ?? true;
43-
const messages: ChatCompletionInputMessage[] = opts?.messages ?? [
44-
{ role: "user", content: "What is the capital of France?" },
45-
];
43+
const exampleMessages: ChatCompletionInputMessage[] =
44+
model.pipeline_tag === "text-generation"
45+
? [{ role: "user", content: "What is the capital of France?" }]
46+
: [
47+
{
48+
role: "user",
49+
content: [
50+
{
51+
type: "image_url",
52+
image_url: {
53+
url: "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg",
54+
},
55+
},
56+
{ type: "text", text: "Describe this image in one sentence." },
57+
],
58+
},
59+
];
60+
const messages = opts?.messages ?? exampleMessages;
4661
const messagesStr = stringifyMessages(messages, { sep: ",\n\t\t", start: "[\n\t\t", end: "\n\t]" });
4762

4863
const config = {
@@ -148,36 +163,6 @@ console.log(chatCompletion.choices[0].message);`,
148163
}
149164
};
150165

151-
export const snippetImageTextToTextGeneration = (model: ModelDataMinimal, accessToken: string): InferenceSnippet => {
152-
if (model.tags.includes("conversational")) {
153-
// Conversational model detected, so we display a code snippet that features the Messages API
154-
return {
155-
content: `import { HfInference } from "@huggingface/inference";
156-
157-
const inference = new HfInference("${accessToken || `{API_TOKEN}`}");
158-
const imageUrl = "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg";
159-
160-
for await (const chunk of inference.chatCompletionStream({
161-
model: "${model.id}",
162-
messages: [
163-
{
164-
"role": "user",
165-
"content": [
166-
{"type": "image_url", "image_url": {"url": imageUrl}},
167-
{"type": "text", "text": "Describe this image in one sentence."},
168-
],
169-
}
170-
],
171-
max_tokens: 500,
172-
})) {
173-
process.stdout.write(chunk.choices[0]?.delta?.content || "");
174-
}`,
175-
};
176-
} else {
177-
return snippetBasic(model, accessToken);
178-
}
179-
};
180-
181166
export const snippetZeroShotClassification = (model: ModelDataMinimal, accessToken: string): InferenceSnippet => ({
182167
content: `async function query(data) {
183168
const response = await fetch(
@@ -307,7 +292,7 @@ export const jsSnippets: Partial<
307292
summarization: snippetBasic,
308293
"feature-extraction": snippetBasic,
309294
"text-generation": snippetTextGeneration,
310-
"image-text-to-text": snippetImageTextToTextGeneration,
295+
"image-text-to-text": snippetTextGeneration,
311296
"text2text-generation": snippetBasic,
312297
"fill-mask": snippetBasic,
313298
"sentence-similarity": snippetBasic,

‎packages/tasks/src/snippets/python.ts

Lines changed: 20 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,24 @@ export const snippetConversational = (
1616
}
1717
): InferenceSnippet[] => {
1818
const streaming = opts?.streaming ?? true;
19-
const messages: ChatCompletionInputMessage[] = opts?.messages ?? [
20-
{ role: "user", content: "What is the capital of France?" },
21-
];
19+
const exampleMessages: ChatCompletionInputMessage[] =
20+
model.pipeline_tag === "text-generation"
21+
? [{ role: "user", content: "What is the capital of France?" }]
22+
: [
23+
{
24+
role: "user",
25+
content: [
26+
{
27+
type: "image_url",
28+
image_url: {
29+
url: "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg",
30+
},
31+
},
32+
{ type: "text", text: "Describe this image in one sentence." },
33+
],
34+
},
35+
];
36+
const messages = opts?.messages ?? exampleMessages;
2237
const messagesStr = stringifyMessages(messages, {
2338
sep: ",\n\t",
2439
start: `[\n\t`,
@@ -121,30 +136,6 @@ print(completion.choices[0].message)`,
121136
}
122137
};
123138

124-
export const snippetConversationalWithImage = (model: ModelDataMinimal, accessToken: string): InferenceSnippet => ({
125-
content: `from huggingface_hub import InferenceClient
126-
127-
client = InferenceClient(api_key="${accessToken || "{API_TOKEN}"}")
128-
129-
image_url = "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
130-
131-
for message in client.chat_completion(
132-
model="${model.id}",
133-
messages=[
134-
{
135-
"role": "user",
136-
"content": [
137-
{"type": "image_url", "image_url": {"url": image_url}},
138-
{"type": "text", "text": "Describe this image in one sentence."},
139-
],
140-
}
141-
],
142-
max_tokens=500,
143-
stream=True,
144-
):
145-
print(message.choices[0].delta.content, end="")`,
146-
});
147-
148139
export const snippetZeroShotClassification = (model: ModelDataMinimal): InferenceSnippet => ({
149140
content: `def query(payload):
150141
response = requests.post(API_URL, headers=headers, json=payload)
@@ -282,7 +273,7 @@ export const pythonSnippets: Partial<
282273
"feature-extraction": snippetBasic,
283274
"text-generation": snippetBasic,
284275
"text2text-generation": snippetBasic,
285-
"image-text-to-text": snippetConversationalWithImage,
276+
"image-text-to-text": snippetConversational,
286277
"fill-mask": snippetBasic,
287278
"sentence-similarity": snippetBasic,
288279
"automatic-speech-recognition": snippetFile,
@@ -306,12 +297,9 @@ export function getPythonInferenceSnippet(
306297
accessToken: string,
307298
opts?: Record<string, unknown>
308299
): InferenceSnippet | InferenceSnippet[] {
309-
if (model.pipeline_tag === "text-generation" && model.tags.includes("conversational")) {
300+
if (model.tags.includes("conversational")) {
310301
// Conversational model detected, so we display a code snippet that features the Messages API
311302
return snippetConversational(model, accessToken, opts);
312-
} else if (model.pipeline_tag === "image-text-to-text" && model.tags.includes("conversational")) {
313-
// Example sending an image to the Message API
314-
return snippetConversationalWithImage(model, accessToken);
315303
} else {
316304
let snippets =
317305
model.pipeline_tag && model.pipeline_tag in pythonSnippets

0 commit comments

Comments
 (0)