feat: configure system prompts by target

Signed-off-by: Alex Fournier <afournier@nvidia.com>

Author: afourniernv

crates/libsy-llm-client/src/lib.rs

@@ -15,6 +15,8 @@
 //! [`switchyard_libsy::Algorithm::run_stream`] and serves every model call the algorithm
 //! offloads, so a host that just wants the answer does not have to drive the step stream
 //! itself.
+//! A host that drives the stream itself can use [`ClientRouter::resolve_call`] to resolve the
+//! selected client and apply target-specific prompts before making each call.
 
 pub mod backend;
 pub mod client;
@@ -30,5 +32,5 @@ pub use client::{ModelConfig, TranslatingLlmClient};
 pub use error::{LlmClientError, Result};
 pub use observation::{LlmCallObservation, RunObservation, RunObserver};
 pub use raw::RawResponse;
-pub use run::{ClientRouter, run};
+pub use run::{ClientRouter, ResolvedLlmCall, run};
 pub use switchyard_translation::RawEventStream;

crates/libsy-llm-client/src/run.rs

@@ -16,7 +16,9 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use parking_lot::Mutex;
-use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive};
+use switchyard_libsy::{
+    Algorithm, CallModel, LibsyError, Result, SystemPromptProcessor, TargetPrompts, drive,
+};
 use switchyard_protocol::{Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient};
 
 use crate::observation::{LlmCallObservation, RunObservation, RunObserver};
@@ -146,7 +148,6 @@ async fn serve(
     routed_calls: Arc<Mutex<RoutedCallWindows>>,
 ) -> Result<()> {
     let span = tracing::Span::current();
-    observability::record_gen_ai_request(&span, &call.request.llm_request);
     if let Some(session_id) = call
         .request
         .metadata
@@ -156,14 +157,21 @@ async fn serve(
         span.record("gen_ai.conversation.id", session_id);
     }
     let target = ModelId::from(call.selected_model_id());
-    let request = call.request.clone();
     let is_answer_call = call.decision.is_answer_call();
     // Resolved before the clock starts: picking the client is Switchyard's work, not
     // the provider's, so it belongs in the routing overhead.
-    let client = clients.route(&target);
+    let resolved = clients.resolve_call(&call.decision, call.request.clone());
+    observability::record_gen_ai_request(
+        &span,
+        &resolved
+            .as_ref()
+            .map(ResolvedLlmCall::request)
+            .unwrap_or(&call.request)
+            .llm_request,
+    );
     let started = Instant::now();
-    let result = match client {
-        Ok(client) => client.call(request).await,
+    let result = match resolved {
+        Ok(call) => call.call().await,
         Err(error) => Err(error),
     }
     .map_err(|source| LibsyError::client_call(target.clone(), source));
@@ -201,7 +209,12 @@ async fn serve(
 /// Cloning is cheap — the mapping is shared, so one router can serve every request.
 #[derive(Clone)]
 pub struct ClientRouter {
-    routing: Arc<Routing>,
+    inner: Arc<ClientRouterInner>,
+}
+
+struct ClientRouterInner {
+    routing: Routing,
+    prompt_processor: SystemPromptProcessor,
 }
 
 enum Routing {
@@ -211,11 +224,40 @@ enum Routing {
     ByModel(HashMap<ModelId, Arc<dyn RoutedLlmClient>>),
 }
 
+/// One normalized model call resolved to its target client and target-specific prompt.
+pub struct ResolvedLlmCall {
+    client: Arc<dyn RoutedLlmClient>,
+    request: Request,
+}
+
+impl ResolvedLlmCall {
+    /// The request that will be sent, including any selected target's system prompt.
+    pub fn request(&self) -> &Request {
+        &self.request
+    }
+
+    /// Perform the resolved call through its selected client.
+    pub async fn call(self) -> std::result::Result<Response, LlmClientError> {
+        self.client.call(self.request).await
+    }
+}
+
 impl ClientRouter {
     /// Build a router over `model name -> client`, for targets spread across providers.
     pub fn new(by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>) -> Self {
+        Self::new_with_target_prompts(by_model, TargetPrompts::default())
+    }
+
+    /// Build a router with system prompts applied to answer calls by selected target.
+    pub fn new_with_target_prompts(
+        by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
+        prompts: TargetPrompts,
+    ) -> Self {
         Self {
-            routing: Arc::new(Routing::ByModel(by_model)),
+            inner: Arc::new(ClientRouterInner {
+                routing: Routing::ByModel(by_model),
+                prompt_processor: SystemPromptProcessor::new(prompts),
+            }),
         }
     }
 
@@ -225,20 +267,51 @@ impl ClientRouter {
     /// backends internally and rejects ones it does not know, so enumerating them here would
     /// only duplicate that.
     pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {
+        Self::single_with_target_prompts(client, TargetPrompts::default())
+    }
+
+    /// A single-client router with system prompts applied by selected target.
+    pub fn single_with_target_prompts(
+        client: Arc<dyn RoutedLlmClient>,
+        prompts: TargetPrompts,
+    ) -> Self {
         Self {
-            routing: Arc::new(Routing::Single(client)),
+            inner: Arc::new(ClientRouterInner {
+                routing: Routing::Single(client),
+                prompt_processor: SystemPromptProcessor::new(prompts),
+            }),
+        }
+    }
+
+    /// Resolve a decision and request into the exact target call a host should perform.
+    ///
+    /// This is the prompt-aware host boundary. It stamps the selected model and prepends
+    /// that target's configured prompt only for answer calls.
+    pub fn resolve_call(
+        &self,
+        decision: &Decision,
+        mut request: Request,
+    ) -> std::result::Result<ResolvedLlmCall, LlmClientError> {
+        let target = decision.selected_model_id();
+        let client = Arc::clone(self.route(target)?);
+        request.llm_request.model = Some(target.to_string());
+        if let Some(prompt) = self.inner.prompt_processor.prompt_for(decision) {
+            switchyard_translation::prepend_system_prompt(&mut request.llm_request, prompt);
         }
+        Ok(ResolvedLlmCall { client, request })
     }
 
-    /// The client that serves `model`.
+    /// The unmodified client that serves `model`.
     ///
     /// Errors with [`LlmClientError::Configuration`] when the router maps models and has no
-    /// entry for this one, rather than silently sending the call to another provider.
+    /// entry for this one, rather than silently sending the call to another provider. This
+    /// lookup does not apply target prompts; hosts serving decisions should use
+    /// [`resolve_call`](Self::resolve_call).
     pub fn route(
         &self,
         model: &ModelId,
     ) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {
-        match self.routing.as_ref() {
+        match &self.inner.routing {
             Routing::Single(client) => Ok(client),
             Routing::ByModel(by_model) => {
                 by_model

crates/libsy-llm-client/tests/target_prompts.rs

@@ -0,0 +1,177 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use parking_lot::Mutex;
+use switchyard_libsy::{Algorithm, Driver, Random, TargetPrompts};
+use switchyard_llm_client::ClientRouter;
+use switchyard_protocol::{
+    ContentBlock, Decision, FormatId, InstructionBlock, LlmClientError, LlmResponse, ModelId,
+    PreservationMetadata, Request, Response, Role, RoutedLlmClient, text_request, text_response,
+};
+
+const JUDGE: &str = "judge/model";
+const WEAK: &str = "weak/model";
+const STRONG: &str = "strong/model";
+
+#[derive(Default)]
+struct RecordingClient {
+    calls: Mutex<Vec<Request>>,
+    overflow: Option<ModelId>,
+}
+
+#[async_trait]
+impl RoutedLlmClient for RecordingClient {
+    async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
+        let model = request.model_id().unwrap_or_default();
+        self.calls.lock().push(request);
+        if self.overflow.as_ref() == Some(&model) {
+            return Err(LlmClientError::ContextWindowExceeded {
+                model,
+                message: "too long".to_string(),
+            });
+        }
+        Ok(Response {
+            llm_response: LlmResponse::Agg(text_response(Some(model.to_string()), "ok")),
+            metadata: None,
+        })
+    }
+}
+
+struct JudgeThenAnswer;
+
+#[async_trait]
+impl Algorithm for JudgeThenAnswer {
+    fn name(&self) -> &str {
+        "judge_then_answer"
+    }
+
+    async fn route(
+        self: Arc<Self>,
+        driver: Driver,
+        request: Request,
+    ) -> switchyard_libsy::Result<Response> {
+        driver
+            .call_model(request.clone(), Decision::new(JUDGE, false))
+            .await?;
+        let decision = Decision::new(STRONG, true);
+        driver.decide(decision.clone()).await?;
+        driver.call_model(request, decision).await
+    }
+}
+
+fn request() -> Request {
+    let mut preservation = PreservationMetadata::default();
+    preservation.requests.insert(
+        "openai_chat".into(),
+        serde_json::json!({
+            "model": "switchyard/test",
+            "messages": [
+                {"role": "system", "content": "client prompt"},
+                {"role": "user", "content": "hi"}
+            ]
+        }),
+    );
+    let mut llm_request = text_request(Some("switchyard/test".to_string()), "hi");
+    llm_request.instructions.push(InstructionBlock {
+        role: Role::System,
+        content: vec![ContentBlock::Text {
+            text: "client prompt".to_string(),
+        }],
+    });
+    llm_request.preservation = preservation;
+    Request {
+        llm_request,
+        raw_request: None,
+        metadata: None,
+    }
+}
+
+fn client_router(client: Arc<dyn RoutedLlmClient>, prompts: TargetPrompts) -> ClientRouter {
+    ClientRouter::single_with_target_prompts(client, prompts)
+}
+
+fn instruction_text(request: &Request) -> Vec<&str> {
+    request
+        .llm_request
+        .instructions
+        .iter()
+        .flat_map(|instruction| instruction.content.iter())
+        .filter_map(|block| match block {
+            ContentBlock::Text { text } => Some(text.as_str()),
+            _ => None,
+        })
+        .collect()
+}
+
+#[tokio::test]
+async fn target_prompts_apply_only_to_answer_calls() -> switchyard_libsy::Result<()> {
+    let client = Arc::new(RecordingClient::default());
+    let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
+    let prompts = TargetPrompts::default()
+        .with(JUDGE, "judge prompt must not be applied")
+        .with(STRONG, "strong prompt");
+
+    switchyard_llm_client::run(
+        Arc::new(JudgeThenAnswer),
+        client_router(routed_client, prompts),
+        request(),
+        None,
+    )
+    .await?;
+
+    let calls = client.calls.lock();
+    assert_eq!(calls.len(), 2);
+    assert_eq!(instruction_text(&calls[0]), ["client prompt"]);
+    assert!(!calls[0].llm_request.preservation.requests.is_empty());
+    assert_eq!(
+        instruction_text(&calls[1]),
+        ["strong prompt", "client prompt"]
+    );
+    assert_eq!(
+        calls[1].llm_request.preservation.requests[&FormatId::from("openai_chat")]["messages"][0],
+        serde_json::json!({"role": "system", "content": "strong prompt"})
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn fallback_call_receives_the_new_targets_prompt() -> switchyard_libsy::Result<()> {
+    let client = Arc::new(RecordingClient {
+        calls: Mutex::new(Vec::new()),
+        overflow: Some(ModelId::from(WEAK)),
+    });
+    let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
+    let prompts = TargetPrompts::default()
+        .with(WEAK, "weak prompt")
+        .with(STRONG, "strong prompt");
+    let algorithm = Random::new(
+        vec![ModelId::from(WEAK), ModelId::from(STRONG)],
+        Some(vec![1.0, 0.0]),
+        Some(1),
+    )?;
+
+    switchyard_llm_client::run(
+        Arc::new(algorithm),
+        client_router(routed_client, prompts),
+        request(),
+        None,
+    )
+    .await?;
+
+    let calls = client.calls.lock();
+    assert_eq!(calls.len(), 2);
+    assert_eq!(calls[0].model_id().as_deref(), Some(WEAK));
+    assert_eq!(
+        instruction_text(&calls[0]),
+        ["weak prompt", "client prompt"]
+    );
+    assert_eq!(calls[1].model_id().as_deref(), Some(STRONG));
+    assert_eq!(
+        instruction_text(&calls[1]),
+        ["strong prompt", "client prompt"]
+    );
+    Ok(())
+}