Skip to content

Commit 0f26700

Browse files
hanouticelinahanouticelina
authored
[Inference] request() returns a request context to avoid redundant makeRequestOptions calls (#1314)
Related to @SBrandeis's comment here #1292 (comment). This PR addresses the original concern about redundant `makeRequestOptions` calls introduced in #1292.The solution implemented here updates the `request` function to return both the response and a _request context_ when needed, allowing provider-specific polling code to reuse this context without redundant calls to `makeRequestOptions`. This differs from the initial suggestion in the comment as each provider implements polling differently with different parameters / response formats. Making a generic `.poll` property would require mixing provider-specific logic into the core request function (we don't want that, right? 😄 ). In the end, we want to keep provider-specific logic isolated in their respective provider files (PR coming today to push that further!).
1 parent e94fd2f commit 0f26700

34 files changed

+311
-285
lines changed

‎packages/inference/README.md

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -572,31 +572,6 @@ await hf.tabularClassification({
572572
})
573573
```
574574

575-
## Custom Calls
576-
577-
For models with custom parameters / outputs.
578-
579-
```typescript
580-
await hf.request({
581-
model: 'my-custom-model',
582-
inputs: 'hello world',
583-
parameters: {
584-
custom_param: 'some magic',
585-
}
586-
})
587-
588-
// Custom streaming call, for models with custom parameters / outputs
589-
for await (const output of hf.streamingRequest({
590-
model: 'my-custom-model',
591-
inputs: 'hello world',
592-
parameters: {
593-
custom_param: 'some magic',
594-
}
595-
})) {
596-
...
597-
}
598-
```
599-
600575
You can use any Chat Completion API-compatible provider with the `chatCompletion` method.
601576

602577
```typescript

‎packages/inference/src/tasks/audio/audioClassification.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import type { AudioClassificationInput, AudioClassificationOutput } from "@huggingface/tasks";
22
import { InferenceOutputError } from "../../lib/InferenceOutputError";
33
import type { BaseArgs, Options } from "../../types";
4-
import { request } from "../custom/request";
4+
import { innerRequest } from "../../utils/request";
55
import type { LegacyAudioInput } from "./utils";
66
import { preparePayload } from "./utils";
77

@@ -16,7 +16,7 @@ export async function audioClassification(
1616
options?: Options
1717
): Promise<AudioClassificationOutput> {
1818
const payload = preparePayload(args);
19-
const res = await request<AudioClassificationOutput>(payload, {
19+
const { data: res } = await innerRequest<AudioClassificationOutput>(payload, {
2020
...options,
2121
task: "audio-classification",
2222
});

‎packages/inference/src/tasks/audio/audioToAudio.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { InferenceOutputError } from "../../lib/InferenceOutputError";
22
import type { BaseArgs, Options } from "../../types";
3-
import { request } from "../custom/request";
3+
import { innerRequest } from "../../utils/request";
44
import type { LegacyAudioInput } from "./utils";
55
import { preparePayload } from "./utils";
66

@@ -37,7 +37,7 @@ export interface AudioToAudioOutput {
3737
*/
3838
export async function audioToAudio(args: AudioToAudioArgs, options?: Options): Promise<AudioToAudioOutput[]> {
3939
const payload = preparePayload(args);
40-
const res = await request<AudioToAudioOutput>(payload, {
40+
const { data: res } = await innerRequest<AudioToAudioOutput>(payload, {
4141
...options,
4242
task: "audio-to-audio",
4343
});

‎packages/inference/src/tasks/audio/automaticSpeechRecognition.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ import type { AutomaticSpeechRecognitionInput, AutomaticSpeechRecognitionOutput
22
import { InferenceOutputError } from "../../lib/InferenceOutputError";
33
import type { BaseArgs, Options, RequestArgs } from "../../types";
44
import { base64FromBytes } from "../../utils/base64FromBytes";
5-
import { request } from "../custom/request";
5+
import { omit } from "../../utils/omit";
6+
import { innerRequest } from "../../utils/request";
67
import type { LegacyAudioInput } from "./utils";
78
import { preparePayload } from "./utils";
8-
import { omit } from "../../utils/omit";
99

1010
export type AutomaticSpeechRecognitionArgs = BaseArgs & (AutomaticSpeechRecognitionInput | LegacyAudioInput);
1111
/**
@@ -17,7 +17,7 @@ export async function automaticSpeechRecognition(
1717
options?: Options
1818
): Promise<AutomaticSpeechRecognitionOutput> {
1919
const payload = await buildPayload(args);
20-
const res = await request<AutomaticSpeechRecognitionOutput>(payload, {
20+
const { data: res } = await innerRequest<AutomaticSpeechRecognitionOutput>(payload, {
2121
...options,
2222
task: "automatic-speech-recognition",
2323
});

‎packages/inference/src/tasks/audio/textToSpeech.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import type { TextToSpeechInput } from "@huggingface/tasks";
22
import { InferenceOutputError } from "../../lib/InferenceOutputError";
33
import type { BaseArgs, Options } from "../../types";
44
import { omit } from "../../utils/omit";
5-
import { request } from "../custom/request";
5+
import { innerRequest } from "../../utils/request";
66
type TextToSpeechArgs = BaseArgs & TextToSpeechInput;
77

88
interface OutputUrlTextToSpeechGeneration {
@@ -22,7 +22,7 @@ export async function textToSpeech(args: TextToSpeechArgs, options?: Options): P
2222
text: args.inputs,
2323
}
2424
: args;
25-
const res = await request<Blob | OutputUrlTextToSpeechGeneration>(payload, {
25+
const { data: res } = await innerRequest<Blob | OutputUrlTextToSpeechGeneration>(payload, {
2626
...options,
2727
task: "text-to-speech",
2828
});

‎packages/inference/src/tasks/custom/request.ts

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import type { InferenceTask, Options, RequestArgs } from "../../types";
2-
import { makeRequestOptions } from "../../lib/makeRequestOptions";
2+
import { innerRequest } from "../../utils/request";
33

44
/**
55
* Primitive to make custom calls to the inference provider
6+
* @deprecated Use specific task functions instead. This function will be removed in a future version.
67
*/
78
export async function request<T>(
89
args: RequestArgs,
@@ -13,35 +14,9 @@ export async function request<T>(
1314
chatCompletion?: boolean;
1415
}
1516
): Promise<T> {
16-
const { url, info } = await makeRequestOptions(args, options);
17-
const response = await (options?.fetch ?? fetch)(url, info);
18-
19-
if (options?.retry_on_error !== false && response.status === 503) {
20-
return request(args, options);
21-
}
22-
23-
if (!response.ok) {
24-
const contentType = response.headers.get("Content-Type");
25-
if (["application/json", "application/problem+json"].some((ct) => contentType?.startsWith(ct))) {
26-
const output = await response.json();
27-
if ([400, 422, 404, 500].includes(response.status) && options?.chatCompletion) {
28-
throw new Error(
29-
`Server ${args.model} does not seem to support chat completion. Error: ${JSON.stringify(output.error)}`
30-
);
31-
}
32-
if (output.error || output.detail) {
33-
throw new Error(JSON.stringify(output.error ?? output.detail));
34-
} else {
35-
throw new Error(output);
36-
}
37-
}
38-
const message = contentType?.startsWith("text/plain;") ? await response.text() : undefined;
39-
throw new Error(message ?? "An error occurred while fetching the blob");
40-
}
41-
42-
if (response.headers.get("Content-Type")?.startsWith("application/json")) {
43-
return await response.json();
44-
}
45-
46-
return (await response.blob()) as T;
17+
console.warn(
18+
"The request method is deprecated and will be removed in a future version of huggingface.js. Use specific task functions instead."
19+
);
20+
const result = await innerRequest<T>(args, options);
21+
return result.data;
4722
}

‎packages/inference/src/tasks/custom/streamingRequest.ts

Lines changed: 5 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
import type { InferenceTask, Options, RequestArgs } from "../../types";
2-
import { makeRequestOptions } from "../../lib/makeRequestOptions";
3-
import type { EventSourceMessage } from "../../vendor/fetch-event-source/parse";
4-
import { getLines, getMessages } from "../../vendor/fetch-event-source/parse";
5-
2+
import { innerStreamingRequest } from "../../utils/request";
63
/**
74
* Primitive to make custom inference calls that expect server-sent events, and returns the response through a generator
5+
* @deprecated Use specific task functions instead. This function will be removed in a future version.
86
*/
97
export async function* streamingRequest<T>(
108
args: RequestArgs,
@@ -15,86 +13,8 @@ export async function* streamingRequest<T>(
1513
chatCompletion?: boolean;
1614
}
1715
): AsyncGenerator<T> {
18-
const { url, info } = await makeRequestOptions({ ...args, stream: true }, options);
19-
const response = await (options?.fetch ?? fetch)(url, info);
20-
21-
if (options?.retry_on_error !== false && response.status === 503) {
22-
return yield* streamingRequest(args, options);
23-
}
24-
if (!response.ok) {
25-
if (response.headers.get("Content-Type")?.startsWith("application/json")) {
26-
const output = await response.json();
27-
if ([400, 422, 404, 500].includes(response.status) && options?.chatCompletion) {
28-
throw new Error(`Server ${args.model} does not seem to support chat completion. Error: ${output.error}`);
29-
}
30-
if (typeof output.error === "string") {
31-
throw new Error(output.error);
32-
}
33-
if (output.error && "message" in output.error && typeof output.error.message === "string") {
34-
/// OpenAI errors
35-
throw new Error(output.error.message);
36-
}
37-
}
38-
39-
throw new Error(`Server response contains error: ${response.status}`);
40-
}
41-
if (!response.headers.get("content-type")?.startsWith("text/event-stream")) {
42-
throw new Error(
43-
`Server does not support event stream content type, it returned ` + response.headers.get("content-type")
44-
);
45-
}
46-
47-
if (!response.body) {
48-
return;
49-
}
50-
51-
const reader = response.body.getReader();
52-
let events: EventSourceMessage[] = [];
53-
54-
const onEvent = (event: EventSourceMessage) => {
55-
// accumulate events in array
56-
events.push(event);
57-
};
58-
59-
const onChunk = getLines(
60-
getMessages(
61-
() => {},
62-
() => {},
63-
onEvent
64-
)
16+
console.warn(
17+
"The streamingRequest method is deprecated and will be removed in a future version of huggingface.js. Use specific task functions instead."
6518
);
66-
67-
try {
68-
while (true) {
69-
const { done, value } = await reader.read();
70-
if (done) {
71-
return;
72-
}
73-
onChunk(value);
74-
for (const event of events) {
75-
if (event.data.length > 0) {
76-
if (event.data === "[DONE]") {
77-
return;
78-
}
79-
const data = JSON.parse(event.data);
80-
if (typeof data === "object" && data !== null && "error" in data) {
81-
const errorStr =
82-
typeof data.error === "string"
83-
? data.error
84-
: typeof data.error === "object" &&
85-
data.error &&
86-
"message" in data.error &&
87-
typeof data.error.message === "string"
88-
? data.error.message
89-
: JSON.stringify(data.error);
90-
throw new Error(`Error forwarded from backend: ` + errorStr);
91-
}
92-
yield data as T;
93-
}
94-
}
95-
events = [];
96-
}
97-
} finally {
98-
reader.releaseLock();
99-
}
19+
yield* innerStreamingRequest(args, options);
10020
}

‎packages/inference/src/tasks/cv/imageClassification.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import type { ImageClassificationInput, ImageClassificationOutput } from "@huggingface/tasks";
22
import { InferenceOutputError } from "../../lib/InferenceOutputError";
33
import type { BaseArgs, Options } from "../../types";
4-
import { request } from "../custom/request";
4+
import { innerRequest } from "../../utils/request";
55
import { preparePayload, type LegacyImageInput } from "./utils";
66

77
export type ImageClassificationArgs = BaseArgs & (ImageClassificationInput | LegacyImageInput);
@@ -15,7 +15,7 @@ export async function imageClassification(
1515
options?: Options
1616
): Promise<ImageClassificationOutput> {
1717
const payload = preparePayload(args);
18-
const res = await request<ImageClassificationOutput>(payload, {
18+
const { data: res } = await innerRequest<ImageClassificationOutput>(payload, {
1919
...options,
2020
task: "image-classification",
2121
});

‎packages/inference/src/tasks/cv/imageSegmentation.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import type { ImageSegmentationInput, ImageSegmentationOutput } from "@huggingface/tasks";
22
import { InferenceOutputError } from "../../lib/InferenceOutputError";
33
import type { BaseArgs, Options } from "../../types";
4-
import { request } from "../custom/request";
4+
import { innerRequest } from "../../utils/request";
55
import { preparePayload, type LegacyImageInput } from "./utils";
66

77
export type ImageSegmentationArgs = BaseArgs & (ImageSegmentationInput | LegacyImageInput);
@@ -15,7 +15,7 @@ export async function imageSegmentation(
1515
options?: Options
1616
): Promise<ImageSegmentationOutput> {
1717
const payload = preparePayload(args);
18-
const res = await request<ImageSegmentationOutput>(payload, {
18+
const { data: res } = await innerRequest<ImageSegmentationOutput>(payload, {
1919
...options,
2020
task: "image-segmentation",
2121
});

‎packages/inference/src/tasks/cv/imageToImage.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import type { ImageToImageInput } from "@huggingface/tasks";
22
import { InferenceOutputError } from "../../lib/InferenceOutputError";
33
import type { BaseArgs, Options, RequestArgs } from "../../types";
44
import { base64FromBytes } from "../../utils/base64FromBytes";
5-
import { request } from "../custom/request";
5+
import { innerRequest } from "../../utils/request";
66

77
export type ImageToImageArgs = BaseArgs & ImageToImageInput;
88

@@ -26,7 +26,7 @@ export async function imageToImage(args: ImageToImageArgs, options?: Options): P
2626
),
2727
};
2828
}
29-
const res = await request<Blob>(reqArgs, {
29+
const { data: res } = await innerRequest<Blob>(reqArgs, {
3030
...options,
3131
task: "image-to-image",
3232
});

‎packages/inference/src/tasks/cv/imageToText.ts

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import type { ImageToTextInput, ImageToTextOutput } from "@huggingface/tasks";
22
import { InferenceOutputError } from "../../lib/InferenceOutputError";
33
import type { BaseArgs, Options } from "../../types";
4-
import { request } from "../custom/request";
4+
import { innerRequest } from "../../utils/request";
55
import type { LegacyImageInput } from "./utils";
66
import { preparePayload } from "./utils";
77

@@ -11,16 +11,14 @@ export type ImageToTextArgs = BaseArgs & (ImageToTextInput | LegacyImageInput);
1111
*/
1212
export async function imageToText(args: ImageToTextArgs, options?: Options): Promise<ImageToTextOutput> {
1313
const payload = preparePayload(args);
14-
const res = (
15-
await request<[ImageToTextOutput]>(payload, {
16-
...options,
17-
task: "image-to-text",
18-
})
19-
)?.[0];
14+
const { data: res } = await innerRequest<[ImageToTextOutput]>(payload, {
15+
...options,
16+
task: "image-to-text",
17+
});
2018

21-
if (typeof res?.generated_text !== "string") {
19+
if (typeof res?.[0]?.generated_text !== "string") {
2220
throw new InferenceOutputError("Expected {generated_text: string}");
2321
}
2422

25-
return res;
23+
return res?.[0];
2624
}

‎packages/inference/src/tasks/cv/objectDetection.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
import { request } from "../custom/request";
2-
import type { BaseArgs, Options } from "../../types";
3-
import { InferenceOutputError } from "../../lib/InferenceOutputError";
41
import type { ObjectDetectionInput, ObjectDetectionOutput } from "@huggingface/tasks";
2+
import { InferenceOutputError } from "../../lib/InferenceOutputError";
3+
import type { BaseArgs, Options } from "../../types";
4+
import { innerRequest } from "../../utils/request";
55
import { preparePayload, type LegacyImageInput } from "./utils";
66

77
export type ObjectDetectionArgs = BaseArgs & (ObjectDetectionInput | LegacyImageInput);
@@ -12,7 +12,7 @@ export type ObjectDetectionArgs = BaseArgs & (ObjectDetectionInput | LegacyImage
1212
*/
1313
export async function objectDetection(args: ObjectDetectionArgs, options?: Options): Promise<ObjectDetectionOutput> {
1414
const payload = preparePayload(args);
15-
const res = await request<ObjectDetectionOutput>(payload, {
15+
const { data: res } = await innerRequest<ObjectDetectionOutput>(payload, {
1616
...options,
1717
task: "object-detection",
1818
});

‎packages/inference/src/tasks/cv/textToImage.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import type { TextToImageInput, TextToImageOutput } from "@huggingface/tasks";
22
import { InferenceOutputError } from "../../lib/InferenceOutputError";
33
import type { BaseArgs, InferenceProvider, Options } from "../../types";
4-
import { omit } from "../../utils/omit";
5-
import { request } from "../custom/request";
64
import { delay } from "../../utils/delay";
5+
import { omit } from "../../utils/omit";
6+
import { innerRequest } from "../../utils/request";
77

88
export type TextToImageArgs = BaseArgs & TextToImageInput;
99

@@ -65,7 +65,7 @@ export async function textToImage(args: TextToImageArgs, options?: TextToImageOp
6565
...getResponseFormatArg(args.provider),
6666
prompt: args.inputs,
6767
};
68-
const res = await request<
68+
const { data: res } = await innerRequest<
6969
| TextToImageOutput
7070
| Base64ImageGeneration
7171
| OutputUrlImageGeneration

0 commit comments

Comments
 (0)