diff --git a/lib/req_llm/provider/defaults.ex b/lib/req_llm/provider/defaults.ex index ea131626..2a26cde2 100644 --- a/lib/req_llm/provider/defaults.ex +++ b/lib/req_llm/provider/defaults.ex @@ -567,12 +567,11 @@ defmodule ReqLLM.Provider.Defaults do |> Req.Request.put_header("authorization", "Bearer #{api_key}") |> Req.Request.register_options(extra_option_keys) |> Req.Request.merge_options( - [ - finch: ReqLLM.Application.finch_name(), - model: model.provider_model_id || model.id, - auth: {:bearer, api_key} - ] ++ - user_opts + finch_option(request) ++ + [ + model: model.provider_model_id || model.id, + auth: {:bearer, api_key} + ] ++ user_opts ) |> ReqLLM.Step.Retry.attach() |> ReqLLM.Step.Error.attach() @@ -583,6 +582,11 @@ defmodule ReqLLM.Provider.Defaults do |> ReqLLM.Step.Fixture.maybe_attach(model, user_opts) end + @spec finch_option(Req.Request.t()) :: keyword() + def finch_option(%Req.Request{} = request) do + [finch: request.options[:finch] || ReqLLM.Application.finch_name()] + end + @doc """ Fetches API key and extra common option keys. """ diff --git a/lib/req_llm/providers/anthropic.ex b/lib/req_llm/providers/anthropic.ex index b83462a7..af45bb27 100644 --- a/lib/req_llm/providers/anthropic.ex +++ b/lib/req_llm/providers/anthropic.ex @@ -316,7 +316,8 @@ defmodule ReqLLM.Providers.Anthropic do |> Req.Request.put_private(:req_llm_model, model) |> maybe_add_beta_header(user_opts) |> Req.Request.merge_options( - [finch: ReqLLM.Application.finch_name(), model: get_api_model_id(model)] ++ user_opts + ReqLLM.Provider.Defaults.finch_option(request) ++ + [model: get_api_model_id(model)] ++ user_opts ) |> ReqLLM.Step.Error.attach() |> ReqLLM.Step.Retry.attach(user_opts) diff --git a/lib/req_llm/providers/azure.ex b/lib/req_llm/providers/azure.ex index 34e4119f..f3127be7 100644 --- a/lib/req_llm/providers/azure.ex +++ b/lib/req_llm/providers/azure.ex @@ -542,7 +542,7 @@ defmodule ReqLLM.Providers.Azure do end) end) |> Req.Request.register_options(registered_option_keys) - |> Req.Request.merge_options([finch: ReqLLM.Application.finch_name()] ++ user_opts) + |> Req.Request.merge_options(ReqLLM.Provider.Defaults.finch_option(request) ++ user_opts) |> ReqLLM.Step.Retry.attach() |> ReqLLM.Step.Error.attach() |> Req.Request.append_response_steps(llm_decode_response: &decode_response/1) diff --git a/lib/req_llm/providers/cohere.ex b/lib/req_llm/providers/cohere.ex index 92726ebf..e6b00bdf 100644 --- a/lib/req_llm/providers/cohere.ex +++ b/lib/req_llm/providers/cohere.ex @@ -79,12 +79,11 @@ defmodule ReqLLM.Providers.Cohere do |> Req.Request.put_header("authorization", "Bearer #{api_key}") |> Req.Request.register_options(extra_option_keys) |> Req.Request.merge_options( - [ - finch: ReqLLM.Application.finch_name(), - model: model.provider_model_id || model.id, - auth: {:bearer, api_key} - ] ++ - user_opts + ReqLLM.Provider.Defaults.finch_option(request) ++ + [ + model: model.provider_model_id || model.id, + auth: {:bearer, api_key} + ] ++ user_opts ) |> ReqLLM.Step.Retry.attach() |> ReqLLM.Step.Error.attach() diff --git a/lib/req_llm/providers/google_vertex.ex b/lib/req_llm/providers/google_vertex.ex index a0353d86..5a92e4bf 100644 --- a/lib/req_llm/providers/google_vertex.ex +++ b/lib/req_llm/providers/google_vertex.ex @@ -349,7 +349,7 @@ defmodule ReqLLM.Providers.GoogleVertex do gcp_creds = Req.Request.get_private(request, :gcp_credentials) request - |> Req.Request.merge_options(finch: ReqLLM.Application.finch_name()) + |> Req.Request.merge_options(ReqLLM.Provider.Defaults.finch_option(request)) |> ReqLLM.Step.Error.attach() |> ReqLLM.Step.Retry.attach() |> Req.Request.append_response_steps(llm_decode_response: &decode_response/1) @@ -381,7 +381,7 @@ defmodule ReqLLM.Providers.GoogleVertex do defp attach_ocr(request, model, gcp_creds, opts) do request - |> Req.Request.merge_options(finch: ReqLLM.Application.finch_name()) + |> Req.Request.merge_options(ReqLLM.Provider.Defaults.finch_option(request)) |> ReqLLM.Step.Error.attach() |> ReqLLM.Step.Retry.attach() |> ReqLLM.Step.Fixture.maybe_attach(model, opts) @@ -390,7 +390,7 @@ defmodule ReqLLM.Providers.GoogleVertex do defp attach_embedding(request, model, gcp_creds, opts) do request - |> Req.Request.merge_options(finch: ReqLLM.Application.finch_name()) + |> Req.Request.merge_options(ReqLLM.Provider.Defaults.finch_option(request)) |> ReqLLM.Step.Error.attach() |> ReqLLM.Step.Retry.attach() |> ReqLLM.Step.Usage.attach(model) diff --git a/lib/req_llm/providers/openai.ex b/lib/req_llm/providers/openai.ex index 9c81c4f7..635fb215 100644 --- a/lib/req_llm/providers/openai.ex +++ b/lib/req_llm/providers/openai.ex @@ -661,12 +661,10 @@ defmodule ReqLLM.Providers.OpenAI do |> maybe_put_authorization_header(credential) |> Req.Request.register_options(extra_option_keys) |> Req.Request.merge_options( - [ - finch: ReqLLM.Application.finch_name(), - model: model.provider_model_id || model.id - ] ++ - auth_req_options(credential) ++ - user_opts + ReqLLM.Provider.Defaults.finch_option(request) ++ + [ + model: model.provider_model_id || model.id + ] ++ auth_req_options(credential) ++ user_opts ) |> ReqLLM.Step.Retry.attach() |> ReqLLM.Step.Error.attach() diff --git a/lib/req_llm/providers/openai_codex.ex b/lib/req_llm/providers/openai_codex.ex index 41cf8823..643c958f 100644 --- a/lib/req_llm/providers/openai_codex.ex +++ b/lib/req_llm/providers/openai_codex.ex @@ -186,11 +186,11 @@ defmodule ReqLLM.Providers.OpenAICodex do |> Req.Request.put_header("originator", originator) |> Req.Request.register_options(extra_option_keys) |> Req.Request.merge_options( - [ - finch: ReqLLM.Application.finch_name(), - model: model.provider_model_id || model.id, - auth: {:bearer, credential.token} - ] ++ user_opts + ReqLLM.Provider.Defaults.finch_option(request) ++ + [ + model: model.provider_model_id || model.id, + auth: {:bearer, credential.token} + ] ++ user_opts ) |> attach_retry(user_opts) |> ReqLLM.Step.Error.attach() diff --git a/test/provider/azure/azure_test.exs b/test/provider/azure/azure_test.exs index 2c98f885..a62d2977 100644 --- a/test/provider/azure/azure_test.exs +++ b/test/provider/azure/azure_test.exs @@ -79,6 +79,21 @@ defmodule ReqLLM.Providers.AzureTest do assert url_string =~ "api-version=2023-05-15" end + test "preserves custom finch from req_http_options" do + {:ok, model} = ReqLLM.model("azure:gpt-4o") + + {:ok, request} = + Azure.prepare_request( + :chat, + model, + "Hello", + base_url: "https://my-resource.openai.azure.com/openai", + req_http_options: [finch: :custom_finch] + ) + + assert request.options[:finch] == :custom_finch + end + test "embedding operation uses correct endpoint" do model = %LLMDB.Model{ id: "text-embedding-3-small", diff --git a/test/providers/openai_test.exs b/test/providers/openai_test.exs index bb847064..79fa9572 100644 --- a/test/providers/openai_test.exs +++ b/test/providers/openai_test.exs @@ -75,6 +75,16 @@ defmodule ReqLLM.Providers.OpenAITest do assert request.method == :post end + test "prepare_request preserves custom finch from req_http_options" do + {:ok, model} = ReqLLM.model("openai:gpt-4-turbo") + context = context_fixture() + + {:ok, request} = + OpenAI.prepare_request(:chat, model, context, req_http_options: [finch: :custom_finch]) + + assert request.options[:finch] == :custom_finch + end + test "prepare_request routes gpt-4o models to Responses API" do {:ok, model} = ReqLLM.model("openai:gpt-4o") context = context_fixture() diff --git a/test/req_llm/step/retry_test.exs b/test/req_llm/step/retry_test.exs index 43c603f9..0bd07b52 100644 --- a/test/req_llm/step/retry_test.exs +++ b/test/req_llm/step/retry_test.exs @@ -103,6 +103,20 @@ defmodule ReqLLM.Step.RetryTest do refute request.options[:retry_delay] end + test "default_attach preserves an existing finch option" do + {:ok, model} = ReqLLM.model("openai:gpt-4") + + request = + ReqLLM.Provider.Defaults.default_attach( + ReqLLM.Providers.OpenAI, + Req.new(finch: :custom_finch), + model, + api_key: "test-key" + ) + + assert request.options[:finch] == :custom_finch + end + test "retry function correctly identifies retryable errors" do {:ok, model} = ReqLLM.model("openai:gpt-4")