Inference: remove the hf-only options now that the API is warm-only (#1192)

Author: julien-c

packages/agents/src/tools/imageToText.ts

@@ -15,12 +15,9 @@ export const imageToTextTool: Tool = {
 		if (typeof data === "string") throw "Input must be a blob.";
 
 		return (
-			await inference.imageToText(
-				{
-					data,
-				},
-				{ wait_for_model: true }
-			)
+			await inference.imageToText({
+				data,
+			})
 		).generated_text;
 	},
 };

packages/agents/src/tools/speechToText.ts

@@ -15,12 +15,9 @@ export const speechToTextTool: Tool = {
 		if (typeof data === "string") throw "Input must be a blob.";
 
 		return (
-			await inference.automaticSpeechRecognition(
-				{
-					data,
-				},
-				{ wait_for_model: true }
-			)
+			await inference.automaticSpeechRecognition({
+				data,
+			})
 		).text;
 	},
 };

packages/agents/src/tools/textToImage.ts

@@ -19,11 +19,8 @@ export const textToImageTool: Tool = {
 		const data = await input;
 		if (typeof data !== "string") throw "Input must be a string.";
 
-		return await inference.textToImage(
-			{
-				inputs: data,
-			},
-			{ wait_for_model: true }
-		);
+		return await inference.textToImage({
+			inputs: data,
+		});
 	},
 };

packages/agents/src/tools/textToSpeech.ts

@@ -19,12 +19,9 @@ export const textToSpeechTool: Tool = {
 		const data = await input;
 		if (typeof data !== "string") throw "Input must be a string.";
 
-		return inference.textToSpeech(
-			{
-				inputs: data,
-				model: "espnet/kan-bayashi_ljspeech_vits",
-			},
-			{ wait_for_model: true }
-		);
+		return inference.textToSpeech({
+			inputs: data,
+			model: "espnet/kan-bayashi_ljspeech_vits",
+		});
 	},
 };

packages/inference/src/lib/makeRequestOptions.ts

@@ -39,8 +39,7 @@ export async function makeRequestOptions(
 	let otherArgs = remainingArgs;
 	const provider = maybeProvider ?? "hf-inference";
 
-	const { forceTask, includeCredentials, taskHint, wait_for_model, use_cache, dont_load_model, chatCompletion } =
-		options ?? {};
+	const { forceTask, includeCredentials, taskHint, chatCompletion } = options ?? {};
 
 	if (endpointUrl && provider !== "hf-inference") {
 		throw new Error(`Cannot use endpointUrl with a third-party provider.`);
@@ -102,18 +101,6 @@ export async function makeRequestOptions(
 		headers["Content-Type"] = "application/json";
 	}
 
-	if (provider === "hf-inference") {
-		if (wait_for_model) {
-			headers["X-Wait-For-Model"] = "true";
-		}
-		if (use_cache === false) {
-			headers["X-Use-Cache"] = "false";
-		}
-		if (dont_load_model) {
-			headers["X-Load-Model"] = "0";
-		}
-	}
-
 	if (provider === "replicate") {
 		headers["Prefer"] = "wait";
 	}

packages/inference/src/tasks/custom/request.ts

@@ -18,11 +18,8 @@ export async function request<T>(
 	const { url, info } = await makeRequestOptions(args, options);
 	const response = await (options?.fetch ?? fetch)(url, info);
 
-	if (options?.retry_on_error !== false && response.status === 503 && !options?.wait_for_model) {
-		return request(args, {
-			...options,
-			wait_for_model: true,
-		});
+	if (options?.retry_on_error !== false && response.status === 503) {
+		return request(args, options);
 	}
 
 	if (!response.ok) {

packages/inference/src/tasks/custom/streamingRequest.ts

@@ -20,11 +20,8 @@ export async function* streamingRequest<T>(
 	const { url, info } = await makeRequestOptions({ ...args, stream: true }, options);
 	const response = await (options?.fetch ?? fetch)(url, info);
 
-	if (options?.retry_on_error !== false && response.status === 503 && !options?.wait_for_model) {
-		return yield* streamingRequest(args, {
-			...options,
-			wait_for_model: true,
-		});
+	if (options?.retry_on_error !== false && response.status === 503) {
+		return yield* streamingRequest(args, options);
 	}
 	if (!response.ok) {
 		if (response.headers.get("Content-Type")?.startsWith("application/json")) {

packages/inference/src/types.ts

@@ -7,26 +7,10 @@ export type ModelId = string;
 
 export interface Options {
 	/**
-	 * (Default: true) Boolean. If a request 503s and wait_for_model is set to false, the request will be retried with the same parameters but with wait_for_model set to true.
+	 * (Default: true) Boolean. If a request 503s, the request will be retried with the same parameters.
 	 */
 	retry_on_error?: boolean;
-	/**
-	 * (Default: true). Boolean. There is a cache layer on Inference API (serverless) to speedup requests we have already seen. Most models can use those results as is as models are deterministic (meaning the results will be the same anyway). However if you use a non deterministic model, you can set this parameter to prevent the caching mechanism from being used resulting in a real new query.
-	 */
-	use_cache?: boolean;
-	/**
-	 * (Default: false). Boolean. Do not load the model if it's not already available.
-	 */
-	dont_load_model?: boolean;
-	/**
-	 * (Default: false). Boolean to use GPU instead of CPU for inference (requires Startup plan at least).
-	 */
-	use_gpu?: boolean;
 
-	/**
-	 * (Default: false) Boolean. If the model is not ready, wait for it instead of receiving 503. It limits the number of requests required to get your inference done. It is advised to only set this flag to true after receiving a 503 error as it will limit hanging in your application to known places.
-	 */
-	wait_for_model?: boolean;
 	/**
 	 * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
 	 */