diff --git a/packages/inference/src/lib/getProviderHelper.ts b/packages/inference/src/lib/getProviderHelper.ts index d7cc87fb59..ff15172d20 100644 --- a/packages/inference/src/lib/getProviderHelper.ts +++ b/packages/inference/src/lib/getProviderHelper.ts @@ -140,6 +140,7 @@ export const PROVIDERS: Record { + const { args } = params; + const { inputs, parameters } = args; + return { + input: { + ...omit(args, ["inputs", "parameters"]), + ...(parameters as Record), + inputs, + }, + version: params.model.includes(":") ? params.model.split(":")[1] : undefined, + }; + } + + // Asynchronous: handles Blob to base64 conversion + async preparePayloadAsync(args: { inputs: Blob } & Record): Promise { + const { inputs, ...restArgs } = args; + const bytes = new Uint8Array(await inputs.arrayBuffer()); + const base64 = base64FromBytes(bytes); + const imageInput = `data:${inputs.type || "image/png"};base64,${base64}`; + return { + ...restArgs, + inputs: imageInput, + }; + } + + // Handle the response from Replicate + override async getResponse(response: ReplicateOutput): Promise { + if ( + typeof response === "object" && + !!response && + "output" in response + ) { + if (Array.isArray(response.output) && response.output.length > 0 && typeof response.output[0] === "string") { + const urlResponse = await fetch(response.output[0]); + return await urlResponse.blob(); + } + if (typeof response.output === "string" && isUrl(response.output)) { + const urlResponse = await fetch(response.output); + return await urlResponse.blob(); + } + } + throw new InferenceClientProviderOutputError("Received malformed response from Replicate image-to-video API"); + } +} diff --git a/packages/inference/test/InferenceClient.spec.ts b/packages/inference/test/InferenceClient.spec.ts index 29cccce8eb..d957d3080b 100644 --- a/packages/inference/test/InferenceClient.spec.ts +++ b/packages/inference/test/InferenceClient.spec.ts @@ -1289,6 +1289,18 @@ describe.skip("InferenceClient", () => { }); expect(res).toBeInstanceOf(Blob); }); + + it("imageToVideo - MeiGen-AI/MeiGen-MultiTalk", async () => { + const res = await client.imageToVideo({ + model: "MeiGen-AI/MeiGen-MultiTalk", + provider: "replicate", + inputs: new Blob([readTestFile("bird_canny.png")], { type: "image/png" }), + parameters: { + prompt: "A bird flying in the sky", + }, + }); + expect(res).toBeInstanceOf(Blob); + }); }, TIMEOUT );