(WIP) Dynamic inference provider mapping #1173
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,4 @@ | ||
| export { HfInference, HfInferenceEndpoint } from "./HfInference"; | ||
| export { InferenceOutputError } from "./lib/InferenceOutputError"; | ||
| export * from "./types"; | ||
| export * from "./tasks"; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| import type { WidgetType } from "@huggingface/tasks"; | ||
| import type { InferenceProvider, InferenceTask, ModelId, Options, RequestArgs } from "../types"; | ||
| import { HF_HUB_URL } from "../config"; | ||
| import { HARDCODED_MODEL_ID_MAPPING } from "../providers/consts"; | ||
|
|
||
| type InferenceProviderMapping = Partial< | ||
| Record<InferenceProvider, { providerId: string; status: "live" | "staging"; task: WidgetType }> | ||
| >; | ||
| const inferenceProviderMappingCache = new Map<ModelId, InferenceProviderMapping>(); | ||
|
|
||
| export async function getProviderModelId( | ||
| params: { | ||
| model: string; | ||
| provider: InferenceProvider; | ||
| }, | ||
| args: RequestArgs, | ||
| options: { | ||
| taskHint?: InferenceTask; | ||
| chatCompletion?: boolean; | ||
| fetch?: Options["fetch"]; | ||
| } = {} | ||
| ): Promise<string> { | ||
| if (params.provider === "hf-inference") { | ||
| return params.model; | ||
| } | ||
| if (!options.taskHint) { | ||
| throw new Error("taskHint must be specified when using a third-party provider"); | ||
| } | ||
| const task: WidgetType = | ||
| options.taskHint === "text-generation" && options.chatCompletion ? "conversational" : options.taskHint; | ||
|
|
||
| // A dict called HARDCODED_MODEL_ID_MAPPING takes precedence in all cases (useful for dev purposes) | ||
| if (HARDCODED_MODEL_ID_MAPPING[params.model]) { | ||
| return HARDCODED_MODEL_ID_MAPPING[params.model]; | ||
| } | ||
|
|
||
| let inferenceProviderMapping: InferenceProviderMapping | null; | ||
| if (inferenceProviderMappingCache.has(params.model)) { | ||
| // eslint-disable-next-line @typescript-eslint/no-non-null-assertion | ||
| inferenceProviderMapping = inferenceProviderMappingCache.get(params.model)!; | ||
| } else { | ||
| inferenceProviderMapping = await (options?.fetch ?? fetch)( | ||
| `${HF_HUB_URL}/api/models/${params.model}?expand[]=inferenceProviderMapping`, | ||
| { | ||
| headers: args.accessToken?.startsWith("hf_") ? { Authorization: `Bearer ${args.accessToken}` } : {}, | ||
| } | ||
| ) | ||
| .then((resp) => resp.json()) | ||
| .then((json) => json.inferenceProviderMapping) | ||
| .catch(() => null); | ||
| } | ||
|
|
||
| if (!inferenceProviderMapping) { | ||
| throw new Error(`We have not been able to find inference provider information for model ${params.model}.`); | ||
| } | ||
|
|
||
| const providerMapping = inferenceProviderMapping[params.provider]; | ||
| if (providerMapping) { | ||
| if (providerMapping.task !== task) { | ||
| throw new Error( | ||
| `Model ${params.model} is not supported for task ${task} and provider ${params.provider}. Supported task: ${providerMapping.task}.` | ||
| ); | ||
| } | ||
| if (providerMapping.status === "staging") { | ||
| console.warn( | ||
| `Model ${params.model} is in staging mode for provider ${params.provider}. Meant for test purposes only.` | ||
| ); | ||
| } | ||
| // TODO: how is it handled server-side if model has multiple tasks (e.g. `text-generation` + `conversational`)? | ||
| return providerMapping.providerId; | ||
| } | ||
|
|
||
| throw new Error(`Model ${params.model} is not supported provider ${params.provider}.`); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| import type { ModelId } from "../types"; | ||
|
|
||
| type ProviderId = string; | ||
|
|
||
| /** | ||
| * If you want to try to run inference for a new model locally before it's registered on huggingface.co | ||
| * for a given Inference Provider, | ||
| * you can add it to the following dictionary, for dev purposes. | ||
| */ | ||
| export const HARDCODED_MODEL_ID_MAPPING: Record<ModelId, ProviderId> = { | ||
| /** | ||
| * "HF model ID" => "Model ID on Inference Provider's side" | ||
| */ | ||
| // "Qwen/Qwen2.5-Coder-32B-Instruct": "Qwen2.5-Coder-32B-Instruct", | ||
| }; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,32 +1,18 @@ | ||
| export const FAL_AI_API_BASE_URL = "https://fal.run"; | ||
|
|
||
|
There was a problem hiding this comment. same comment as in https://github.com/huggingface/huggingface_hub/pull/2836/files#r1942755169, i would for now keep the mappings (but make them empty) That way we have slightly less breaking changes too, potentially There was a problem hiding this comment. @julien-c I reverted to keep both the hard-coded logic and the new dynamic mapping. Mapping from the Hub takes precedence over the hardcoded one. |
||
| /** | ||
| * See the registered mapping of HF model ID => Fal model ID here: | ||
| * | ||
| * https://huggingface.co/api/partners/fal-ai/models | ||
| * | ||
| * This is a publicly available mapping. | ||
| * | ||
| * If you want to try to run inference for a new model locally before it's registered on huggingface.co, | ||
| * you can add it to the dictionary "HARDCODED_MODEL_ID_MAPPING" in consts.ts, for dev purposes. | ||
| * | ||
| * - If you work at Fal and want to update this mapping, please use the model mapping API we provide on huggingface.co | ||
| * - If you're a community member and want to add a new supported HF model to Fal, please open an issue on the present repo | ||
| * and we will tag Fal team members. | ||
| * | ||
| * Thanks! | ||
| */ | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,30 +1,18 @@ | ||
| export const REPLICATE_API_BASE_URL = "https://api.replicate.com"; | ||
|
|
||
| /** | ||
| * See the registered mapping of HF model ID => Replicate model ID here: | ||
| * | ||
| * https://huggingface.co/api/partners/replicate/models | ||
| * | ||
| * This is a publicly available mapping. | ||
| * | ||
| * If you want to try to run inference for a new model locally before it's registered on huggingface.co, | ||
| * you can add it to the dictionary "HARDCODED_MODEL_ID_MAPPING" in consts.ts, for dev purposes. | ||
| * | ||
| * - If you work at Replicate and want to update this mapping, please use the model mapping API we provide on huggingface.co | ||
| * - If you're a community member and want to add a new supported HF model to Replicate, please open an issue on the present repo | ||
| * and we will tag Replicate team members. | ||
| * | ||
| * Thanks! | ||
| */ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,25 +1,18 @@ | ||
| export const SAMBANOVA_API_BASE_URL = "https://api.sambanova.ai"; | ||
|
|
||
| /** | ||
| * See the registered mapping of HF model ID => Sambanova model ID here: | ||
| * | ||
| * https://huggingface.co/api/partners/sambanova/models | ||
| * | ||
| * This is a publicly available mapping. | ||
| * | ||
| * If you want to try to run inference for a new model locally before it's registered on huggingface.co, | ||
| * you can add it to the dictionary "HARDCODED_MODEL_ID_MAPPING" in consts.ts, for dev purposes. | ||
| * | ||
| * - If you work at Sambanova and want to update this mapping, please use the model mapping API we provide on huggingface.co | ||
| * - If you're a community member and want to add a new supported HF model to Sambanova, please open an issue on the present repo | ||
| * and we will tag Sambanova team members. | ||
| * | ||
| * Thanks! | ||
| */ |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
i think this is ok @Wauplin