Skip to main content

switchyard_llm_client/
run.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Drive a libsy algorithm to completion, making the model calls it offloads.
5//!
6//! [`switchyard_libsy::Algorithm::run_stream`] is the whole libsy API: it yields a stream of
7//! steps and expects its consumer to serve every offloaded model call. [`run()`] is that
8//! consumer — it drives the stream with [`switchyard_libsy::drive`], hands routing-time calls to
9//! a [`RoutedLlmClient`], and serves the terminal routing outcome.
10//!
11//! libsy owns the stream mechanics; what this module adds is per-target request preparation,
12//! ordered candidate fallback, and the `libsy.client_call` span around each candidate. Each
13//! completion candidate exhausts its backend retry budget before fallback advances. Routing
14//! calls stop on the first candidate's failure. A timeout stops either kind of call.
15//!
16//! A Responses continuation can refer to state through `previous_response_id` or `conversation`.
17//! [`ClientRouter`] sends native Responses state back to its provider; for Chat or Anthropic
18//! targets it retains the canonical history needed to materialize the continuation.
19
20use std::collections::HashMap;
21use std::sync::Arc;
22use std::time::Instant;
23
24use futures::{StreamExt, TryStreamExt, stream};
25use http::StatusCode;
26use parking_lot::Mutex;
27use serde_json::{Value, json};
28use switchyard_libsy::{
29    Algorithm, CallModel, LibsyError, OutcomeMetadata, Result, RoutingOutcome, RuntimeModels, drive,
30};
31use switchyard_protocol::{
32    AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message,
33    ModelId, Request, Response, ResponseAccumulator, RoutedLlmClient, RoutingFallbackReason,
34    WireFormat,
35};
36use switchyard_translation::prepare_request_for_target;
37
38use crate::observation::{LlmCallObservation, RunObservation, RunObserver};
39use crate::{metrics, observability};
40
41/// Run one request to completion, serving every offloaded model call with `client`.
42///
43/// Returns the model selected by the algorithm and the final [`Response`]. `observer`, when
44/// present, receives metadata for the routing outcome, each completed routing or answer call,
45/// and the routing overhead.
46///
47/// `clients` resolves each offloaded call to the client for the target the algorithm
48/// selected — an algorithm may route among targets served by different providers, so this is
49/// a per-call lookup, not one client for the whole run. Use
50/// [`ClientRouter::single`](ClientRouter::single) when one client serves every target.
51///
52/// Routing calls are buffered; a client failure stops the request before the algorithm continues.
53/// Once routing completes, non-timeout failures may try the outcome's ordered fallback candidates.
54pub async fn run(
55    algorithm: Arc<dyn Algorithm>,
56    clients: ClientRouter,
57    request: Request,
58    models: Arc<RuntimeModels>,
59    observer: Option<RunObserver>,
60) -> Result<(ModelId, Response)> {
61    let algorithm_name = algorithm.name().to_string();
62    let run_started = Instant::now();
63    let routing_clients = clients.clone();
64    // This says if we have an observer, put Some(..) in routing_observations.
65    // No observer means we don't want any routing_observations.
66    let routing_observations = observer.as_ref().map(|_| Arc::new(Mutex::new(Vec::new())));
67    let outcome = match clients.stored_state_owner(&request) {
68        Some(owner) => Ok(continue_on(owner, &algorithm_name, request)),
69        None => {
70            drive(algorithm, request, models, {
71                let routing_observations = routing_observations.clone();
72                move |call| serve(routing_clients.clone(), call, routing_observations.clone())
73            })
74            .await
75        }
76    };
77    let answered_model = outcome
78        .as_ref()
79        .ok()
80        .and_then(|outcome| outcome.response.as_ref())
81        .and_then(Response::served_model);
82    emit_routing_observations(&observer, &routing_observations, answered_model);
83    let outcome = outcome?;
84    let overhead = run_started.elapsed();
85    metrics::record_routing_overhead(&algorithm_name, overhead);
86
87    let selected_model_id = outcome.selected_model_id()?.clone();
88    if let Some(observer) = &observer
89        && let Some(metadata) = outcome.metadata
90    {
91        observer(RunObservation::Outcome(metadata));
92    }
93    let result = if let Some(response) = outcome.response {
94        let result = clients.remember_state_owner(&outcome.request, response);
95        let served_model = result
96            .as_ref()
97            .ok()
98            .and_then(Response::served_model)
99            .unwrap_or(&selected_model_id);
100        let served_model = served_model.clone();
101        metrics::observe_routed_request(&algorithm_name, &served_model, None, result)
102    } else {
103        let observe = |observation| {
104            if let Some(observer) = &observer {
105                observer(RunObservation::AnswerCall(observation));
106            }
107        };
108        call_first_available(
109            &clients,
110            &algorithm_name,
111            &outcome.request,
112            &outcome.selected_model_ids,
113            &observe,
114        )
115        .await
116        .and_then(|response| clients.remember_state_owner(&outcome.request, response))
117    };
118    if let Some(observer) = &observer {
119        observer(RunObservation::RoutingOverhead(overhead));
120    }
121    result.map(|response| (selected_model_id, response))
122}
123
124/// Run an algorithm to a routing decision without serving its terminal completion.
125///
126/// Routing-time calls are served normally. The returned request is prepared only for the selected
127/// target; use [`run`] to execute selected and fallback candidates.
128pub async fn decide(
129    algorithm: Arc<dyn Algorithm>,
130    clients: ClientRouter,
131    request: Request,
132    models: Arc<RuntimeModels>,
133) -> Result<RoutingOutcome> {
134    let routing_clients = clients.clone();
135    let mut outcome = match clients.stored_state_owner(&request) {
136        Some(owner) => continue_on(owner, algorithm.name(), request),
137        None => {
138            drive(algorithm, request, models, move |call| {
139                serve(routing_clients.clone(), call, None)
140            })
141            .await?
142        }
143    };
144    let selected_model_id = outcome.selected_model_id()?.clone();
145    outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id);
146    outcome.response = outcome
147        .response
148        .map(|response| clients.remember_state_owner(&outcome.request, response))
149        .transpose()?;
150    Ok(outcome)
151}
152
153/// Emits completed routing calls after the outcome reveals whether one response became the answer.
154fn emit_routing_observations(
155    observer: &Option<RunObserver>,
156    observations: &Option<Arc<Mutex<Vec<LlmCallObservation>>>>,
157    answered_model: Option<&ModelId>,
158) {
159    let (Some(observer), Some(observations)) = (observer, observations) else {
160        return;
161    };
162    let mut answer_observed = false;
163    for observation in observations.lock().drain(..) {
164        if !answer_observed && answered_model == Some(&observation.selected_model) {
165            answer_observed = true;
166            observer(RunObservation::AnswerCall(observation));
167        } else {
168            observer(RunObservation::LlmCall(observation));
169        }
170    }
171}
172
173/// Serve one offloaded call and fulfill its promise.
174///
175/// A client failure stops the driver before the algorithm can issue another call.
176async fn serve(
177    clients: ClientRouter,
178    call: CallModel,
179    observations: Option<Arc<Mutex<Vec<LlmCallObservation>>>>,
180) -> Result<()> {
181    let observe = |observation| {
182        if let Some(observations) = &observations {
183            observations.lock().push(observation);
184        }
185    };
186    let target = call.models.first().ok_or(LibsyError::NoTargets)?;
187    let request = clients.prepare_routing_request(call.request.clone(), target);
188    let response = call_one(
189        &clients,
190        target,
191        request,
192        &call.algorithm,
193        &observe,
194        0,
195        call.models.len(),
196        true,
197    )
198    .await?;
199    call.respond(Ok(response))
200}
201
202/// Try candidates in order until one succeeds or a failure stops fallback.
203async fn call_first_available(
204    clients: &ClientRouter,
205    algorithm: &str,
206    request: &Request,
207    models: &[ModelId],
208    observe: &(dyn Fn(LlmCallObservation) + Send + Sync),
209) -> Result<Response> {
210    for (index, target) in models.iter().enumerate() {
211        let request = clients.prepare_completion_request(request.clone(), target);
212        match call_one(
213            clients,
214            target,
215            request,
216            algorithm,
217            observe,
218            index,
219            models.len(),
220            false,
221        )
222        .await
223        {
224            Ok(response) => return Ok(response),
225            Err(error) if index + 1 == models.len() => return Err(error),
226            Err(error) => match fallback_reason(&error) {
227                Some(reason) => tracing::info!(
228                    from = %target,
229                    to = %models[index + 1],
230                    reason = reason.as_str(),
231                    "model call failed; trying next candidate"
232                ),
233                None => return Err(error),
234            },
235        }
236    }
237    Err(LibsyError::NoTargets)
238}
239
240/// Call one candidate model and record its observation and span.
241#[allow(clippy::too_many_arguments)]
242#[tracing::instrument(
243    target = "libsy",
244    name = "libsy.client_call",
245    skip_all,
246    fields(
247        algorithm = algorithm,
248        switchyard.algorithm = algorithm,
249        switchyard.candidate = index + 1,
250        switchyard.candidate_count = count,
251        selected_model = %model_id,
252        otel.kind = "client",
253        otel.name = %format_args!("chat {model_id}"),
254        openinference.span.kind = "LLM",
255        gen_ai.operation.name = "chat",
256        gen_ai.request.model = %model_id,
257        gen_ai.request.stream = tracing::field::Empty,
258        gen_ai.request.temperature = tracing::field::Empty,
259        gen_ai.request.top_p = tracing::field::Empty,
260        gen_ai.request.top_k = tracing::field::Empty,
261        gen_ai.request.max_tokens = tracing::field::Empty,
262        gen_ai.request.reasoning.level = tracing::field::Empty,
263        gen_ai.output.type = tracing::field::Empty,
264        gen_ai.conversation.id = tracing::field::Empty,
265        server.address = tracing::field::Empty,
266        server.port = tracing::field::Empty,
267        gen_ai.response.id = tracing::field::Empty,
268        gen_ai.response.model = tracing::field::Empty,
269        gen_ai.usage.input_tokens = tracing::field::Empty,
270        gen_ai.usage.output_tokens = tracing::field::Empty,
271        gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
272        gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
273        gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
274        outcome = tracing::field::Empty,
275        otel.status_code = tracing::field::Empty,
276        error.type = tracing::field::Empty,
277        error = tracing::field::Empty,
278    )
279)]
280async fn call_one(
281    clients: &ClientRouter,
282    model_id: &ModelId,
283    request: Request,
284    algorithm: &str,
285    observe: &(dyn Fn(LlmCallObservation) + Send + Sync),
286    // index is for span log
287    index: usize,
288    // count is for span log
289    count: usize,
290    buffer: bool,
291) -> Result<Response> {
292    let span = tracing::Span::current();
293    observability::record_gen_ai_request(&span, &request.llm_request);
294    if let Some(session_id) = request
295        .metadata
296        .as_ref()
297        .and_then(|metadata| metadata.session_id.as_deref())
298    {
299        span.record("gen_ai.conversation.id", session_id);
300    }
301    // Resolved before the clock starts: picking the client is Switchyard's work, not
302    // the provider's, so it belongs in the routing overhead.
303    let client = clients.route(model_id);
304    let started = Instant::now();
305    let result = async {
306        let mut response = client?.call(request).await?;
307        if buffer && let LlmResponse::Stream(chunks) = response.llm_response {
308            response.llm_response = LlmResponse::Stream(buffer_routing_stream(chunks).await?);
309        }
310        Ok(response)
311    }
312    .await
313    .map_err(|source| LibsyError::client_call(model_id.clone(), source));
314    let duration = started.elapsed();
315
316    let result = result.map(|mut response| {
317        response.set_served_model(model_id);
318        response
319    });
320    let result = observability::observe_client_call(result);
321    let result = if !buffer {
322        metrics::observe_routed_request(algorithm, model_id, Some(duration), result)
323    } else {
324        result
325    };
326    observe(LlmCallObservation {
327        selected_model: model_id.clone(),
328        is_success: result.is_ok(),
329        duration,
330        usage: result
331            .as_ref()
332            .ok()
333            .and_then(|response| response.llm_response.as_agg())
334            .map(|response| response.usage.clone()),
335    });
336    result
337}
338
339// Preserve signed provider events while checking the complete routing response.
340async fn buffer_routing_stream(
341    mut chunks: LlmResponseStream,
342) -> std::result::Result<LlmResponseStream, LlmClientError> {
343    let mut events = Vec::new();
344    while let Some(event) = chunks.try_next().await? {
345        for chunk in event.normalized() {
346            match chunk {
347                LlmResponseChunk::DecodeError { message } => {
348                    return Err(LlmClientError::ResponseTranslation(message.clone()));
349                }
350                LlmResponseChunk::StreamError { message } => {
351                    return Err(LlmClientError::UpstreamHttp {
352                        status: StatusCode::BAD_GATEWAY,
353                        body: message.clone(),
354                    });
355                }
356                _ => {}
357            }
358        }
359        events.push(event);
360    }
361    Ok(stream::iter(events.into_iter().map(Ok)).boxed())
362}
363
364/// Whether a failed candidate is worth routing around.
365fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
366    let LibsyError::ClientCall { source, .. } = error else {
367        return None;
368    };
369    match source {
370        LlmClientError::ContextWindowExceeded { .. } => Some(RoutingFallbackReason::ContextWindow),
371        LlmClientError::Transport { .. } => Some(RoutingFallbackReason::Unavailable),
372        // A policy denial can be specific to one provider. Preserve its HTTP
373        // error, but allow another candidate to serve the request.
374        LlmClientError::UpstreamHttp { status, body }
375            if *status == StatusCode::BAD_REQUEST
376                && serde_json::from_str::<serde_json::Value>(body).is_ok_and(|value| {
377                    value["error"]["code"].as_str() == Some("content_policy_violation")
378                }) =>
379        {
380            Some(RoutingFallbackReason::Unavailable)
381        }
382        LlmClientError::UpstreamHttp { status, .. }
383            if matches!(
384                *status,
385                StatusCode::FORBIDDEN | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS
386            ) || status.is_server_error() =>
387        {
388            Some(RoutingFallbackReason::Unavailable)
389        }
390        _ => None,
391    }
392}
393
394/// Route to `owner` without fallbacks and record why the algorithm did not run.
395fn continue_on(owner: StateOwner, algorithm: &str, mut request: Request) -> RoutingOutcome {
396    tracing::debug!(
397        target = %owner.model,
398        trigger = owner.trigger,
399        "routing Responses continuation to the model that holds its state"
400    );
401    if let Some(history) = owner.history {
402        let mut messages = Vec::with_capacity(history.len + request.llm_request.messages.len());
403        history.extend(&mut messages);
404        messages.append(&mut request.llm_request.messages);
405        request.llm_request.messages = messages;
406    }
407    let mut outcome = RoutingOutcome::route_to(owner.model, Vec::new(), request);
408    outcome.metadata = Some(OutcomeMetadata::new(
409        algorithm.to_string(),
410        Some(json!({"source": "stored_state", "trigger": owner.trigger})),
411    ));
412    outcome
413}
414
415/// The model that served the stored state and the request field that refers to it.
416struct StateOwner {
417    model: ModelId,
418    trigger: &'static str,
419    history: Option<Arc<MessageHistory>>,
420}
421
422struct MessageHistory {
423    parent: Option<Arc<Self>>,
424    segment: Arc<[Message]>,
425    len: usize,
426}
427
428impl MessageHistory {
429    fn new(parent: Option<Arc<Self>>, segment: Arc<[Message]>) -> Self {
430        let len = parent.as_ref().map_or(0, |history| history.len) + segment.len();
431        Self {
432            parent,
433            segment,
434            len,
435        }
436    }
437
438    fn extend(&self, messages: &mut Vec<Message>) {
439        let mut segments = Vec::new();
440        let mut history = Some(self);
441        while let Some(node) = history {
442            segments.push(node.segment.as_ref());
443            history = node.parent.as_deref();
444        }
445        for segment in segments.into_iter().rev() {
446            messages.extend_from_slice(segment);
447        }
448    }
449}
450
451#[derive(Clone)]
452struct StoredState {
453    model: ModelId,
454    /// `None` means the provider owns the state and only routing must be pinned.
455    history: Option<Arc<MessageHistory>>,
456}
457
458#[derive(Clone)]
459struct CanonicalInput {
460    parent: Option<Arc<MessageHistory>>,
461    messages: Arc<[Message]>,
462}
463
464/// Stored Responses IDs and conversation IDs, with their provider or canonical state.
465///
466/// Keep earlier response IDs because a client can continue from any stored response.
467/// Reject new IDs at capacity rather than forgetting where an earlier response is stored.
468#[derive(Default)]
469struct StateOwners {
470    by_id: HashMap<String, StoredState>,
471}
472
473const MAX_STATE_OWNERS: usize = 65_536;
474
475impl StateOwners {
476    fn owner(&self, id: &str) -> Option<&StoredState> {
477        self.by_id.get(id)
478    }
479
480    fn remember(
481        &mut self,
482        response_id: Option<&str>,
483        conversation_id: Option<&str>,
484        model: &ModelId,
485        history: Option<Arc<MessageHistory>>,
486    ) -> std::result::Result<(), LlmClientError> {
487        let materialized = history.is_some();
488        let ids = [
489            response_id,
490            conversation_id.filter(|id| Some(*id) != response_id),
491        ];
492        let mut new_ids = 0;
493        for id in ids.into_iter().flatten() {
494            match self.by_id.get(id) {
495                Some(owner) if owner.model != *model && !materialized => {
496                    return Err(LlmClientError::ResponseStateConflict);
497                }
498                None => new_ids += 1,
499                _ => {}
500            }
501        }
502        if self.by_id.len() + new_ids > MAX_STATE_OWNERS {
503            return Err(LlmClientError::ResponseStateLimitExceeded {
504                limit: MAX_STATE_OWNERS,
505            });
506        }
507        let state = StoredState {
508            model: model.clone(),
509            history,
510        };
511        if let Some(id) = response_id {
512            if materialized {
513                self.by_id.insert(id.to_owned(), state.clone());
514            } else {
515                self.by_id
516                    .entry(id.to_owned())
517                    .or_insert_with(|| state.clone());
518            }
519        }
520        if let Some(id) = conversation_id {
521            self.by_id.entry(id.to_owned()).or_insert(state);
522        }
523        Ok(())
524    }
525}
526
527/// Read the Responses `conversation` ID from a string or an object.
528fn conversation_id(fields: &serde_json::Map<String, Value>) -> Option<&str> {
529    let conversation = fields.get("conversation")?;
530    conversation
531        .as_str()
532        .or_else(|| conversation.get("id").and_then(Value::as_str))
533}
534
535/// Resolves a routed call's selected model to the client that serves it.
536///
537/// An algorithm routes among named targets; which provider each target lives on is the
538/// host's concern, and two targets in one run may sit on different providers. A router owns
539/// that mapping. It is *not* itself a client: it hands back a [`RoutedLlmClient`] and the
540/// caller makes the call.
541///
542/// Cloning is cheap — the mapping is shared, so one router can serve every request.
543#[derive(Clone)]
544pub struct ClientRouter {
545    inner: Arc<ClientRouting>,
546}
547
548struct ClientRouting {
549    routing: Routing,
550    target_prompts: HashMap<ModelId, String>,
551    routing_answer_target: Option<ModelId>,
552    /// Provider-owned routing pins and materialized cross-format history.
553    state_owners: Mutex<StateOwners>,
554    /// Native Responses state only needs a routing pin when answer targets span clients.
555    track_provider_state: bool,
556}
557
558enum Routing {
559    /// One client serves every model.
560    Single(Arc<dyn RoutedLlmClient>),
561    /// Each model is served by the client configured for it.
562    ByModel(HashMap<ModelId, Arc<dyn RoutedLlmClient>>),
563}
564
565impl ClientRouter {
566    /// Build a router over `model name -> client`, for targets spread across providers.
567    pub fn new(by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>) -> Self {
568        Self::new_with_target_prompts(by_model, HashMap::new(), None)
569    }
570
571    /// Build a router with target prompts used by [`run`] and [`decide`].
572    ///
573    /// `routing_answer_target` identifies a routing-time call whose response may become the
574    /// answer. Pass `None` when routing only selects a later completion target. Prompt keys and
575    /// `routing_answer_target` use the resolved model IDs stored in `by_model`.
576    /// This constructor treats every model in `by_model` as a possible completion target.
577    /// Use [`Self::new_with_completion_targets`] to exclude classifier-only models from the
578    /// comparison of completion clients.
579    pub fn new_with_target_prompts(
580        by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
581        target_prompts: HashMap<ModelId, String>,
582        routing_answer_target: Option<ModelId>,
583    ) -> Self {
584        let completion_targets = by_model.keys().cloned().collect::<Vec<_>>();
585        Self::new_with_completion_targets(
586            by_model,
587            target_prompts,
588            routing_answer_target,
589            &completion_targets,
590        )
591    }
592
593    /// Build a router with an explicit list of models that can answer requests.
594    ///
595    /// `by_model` contains every callable model, including classifiers. Only the model IDs in
596    /// `completion_targets` determine whether native Responses state needs a routing pin.
597    /// Cross-format state is recorded lazily when a Responses request uses a Chat or Anthropic
598    /// target. `target_prompts` and `routing_answer_target` have the same meaning as in
599    /// [`Self::new_with_target_prompts`].
600    pub fn new_with_completion_targets(
601        by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
602        target_prompts: HashMap<ModelId, String>,
603        routing_answer_target: Option<ModelId>,
604        completion_targets: &[ModelId],
605    ) -> Self {
606        let mut clients = completion_targets
607            .iter()
608            .filter_map(|model| by_model.get(model));
609        let spans_providers = clients
610            .next()
611            .is_some_and(|first| clients.any(|client| !Arc::ptr_eq(first, client)));
612        Self {
613            inner: Arc::new(ClientRouting {
614                routing: Routing::ByModel(by_model),
615                target_prompts,
616                routing_answer_target,
617                state_owners: Mutex::default(),
618                track_provider_state: spans_providers,
619            }),
620        }
621    }
622
623    /// A router that serves every model with one client — the single-provider case.
624    ///
625    /// [`TranslatingLlmClient`](crate::TranslatingLlmClient) already maps model names to
626    /// backends internally and rejects ones it does not know, so enumerating them here would
627    /// only duplicate that.
628    pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {
629        Self {
630            inner: Arc::new(ClientRouting {
631                routing: Routing::Single(client),
632                target_prompts: HashMap::new(),
633                routing_answer_target: None,
634                state_owners: Mutex::default(),
635                track_provider_state: false,
636            }),
637        }
638    }
639
640    /// The client that serves `model`.
641    ///
642    /// Errors with [`LlmClientError::Configuration`] when the router maps models and has no
643    /// entry for this one, rather than silently sending the call to another provider.
644    pub fn route(
645        &self,
646        model: &ModelId,
647    ) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {
648        match &self.inner.routing {
649            Routing::Single(client) => Ok(client),
650            Routing::ByModel(by_model) => {
651                by_model
652                    .get(model)
653                    .ok_or_else(|| LlmClientError::Configuration {
654                        message: format!("no llm client is configured for model {model:?}"),
655                    })
656            }
657        }
658    }
659
660    /// Return the recorded model for the requested response or conversation ID.
661    fn stored_state_owner(&self, request: &Request) -> Option<StateOwner> {
662        let fields = &request.llm_request.extensions.fields;
663        let (trigger, id) = match fields.get("previous_response_id").and_then(Value::as_str) {
664            Some(id) => ("previous_response_id", id),
665            None => ("conversation", conversation_id(fields)?),
666        };
667        let state = self.inner.state_owners.lock().owner(id)?.clone();
668        Some(StateOwner {
669            model: state.model,
670            trigger,
671            history: state.history,
672        })
673    }
674
675    fn canonical_input(&self, request: &Request) -> CanonicalInput {
676        let parent = request
677            .llm_request
678            .extensions
679            .fields
680            .get("previous_response_id")
681            .and_then(Value::as_str)
682            .and_then(|id| self.inner.state_owners.lock().owner(id)?.history.clone());
683        let parent_len = parent.as_ref().map_or(0, |history| history.len);
684        let (parent, messages) = match request.llm_request.messages.get(parent_len..) {
685            Some(messages) => (parent, messages.to_vec()),
686            None => (None, request.llm_request.messages.clone()),
687        };
688        CanonicalInput {
689            parent,
690            messages: Arc::from(messages),
691        }
692    }
693
694    /// Record provider-owned Responses state or canonical history for a cross-format response.
695    fn remember_state_owner(&self, request: &Request, mut response: Response) -> Result<Response> {
696        let Some(model) = response.served_model().cloned() else {
697            return Ok(response);
698        };
699        let fields = &request.llm_request.extensions.fields;
700        let store = fields.get("store").and_then(Value::as_bool) != Some(false);
701        let conversation = conversation_id(fields).map(str::to_owned);
702        let responses_format = WireFormat::OpenAiResponses.into();
703        let responses_request = request
704            .llm_request
705            .preservation
706            .requests
707            .contains_key(&responses_format)
708            || fields.contains_key("previous_response_id")
709            || fields.contains_key("conversation");
710        if !responses_request && !self.inner.track_provider_state {
711            return Ok(response);
712        }
713        let canonical_input = responses_request.then(|| self.canonical_input(request));
714        response.llm_response = match response.llm_response {
715            LlmResponse::Agg(agg) => {
716                if let Some(body) = agg.preservation.responses.get(&responses_format) {
717                    if self.inner.track_provider_state {
718                        self.remember_response(body, &model, store, conversation.as_deref())
719                            .map_err(|error| LibsyError::client_call(model.clone(), error))?;
720                    }
721                } else if let Some(input) = &canonical_input {
722                    self.remember_canonical_response(&agg, &model, store, input)
723                        .map_err(|error| LibsyError::client_call(model.clone(), error))?;
724                }
725                LlmResponse::Agg(agg)
726            }
727            LlmResponse::Stream(stream) => {
728                let router = self.clone();
729                LlmResponse::Stream(Box::pin(futures_util::stream::try_unfold(
730                    (
731                        stream,
732                        router,
733                        model,
734                        conversation,
735                        canonical_input,
736                        ResponseAccumulator::new(),
737                        false,
738                        true,
739                    ),
740                    move |(
741                        mut stream,
742                        router,
743                        model,
744                        conversation,
745                        canonical_input,
746                        mut accumulator,
747                        mut native_responses,
748                        mut valid,
749                    )| async move {
750                        let Some(event) = stream.next().await else {
751                            if let Some(input) = &canonical_input
752                                && !native_responses
753                                && valid
754                            {
755                                router.remember_canonical_response(
756                                    &accumulator.finish(),
757                                    &model,
758                                    store,
759                                    input,
760                                )?;
761                            }
762                            return Ok(None);
763                        };
764                        let event = event?;
765                        if let Some(preserved) = event.preservation()
766                            && preserved.source().as_str() == WireFormat::OpenAiResponses.as_str()
767                        {
768                            native_responses = true;
769                            if router.inner.track_provider_state
770                                && let Some(body) = preserved.raw().get("response")
771                            {
772                                router.remember_response(
773                                    body,
774                                    &model,
775                                    store,
776                                    conversation.as_deref(),
777                                )?;
778                            }
779                        }
780                        if responses_request {
781                            for chunk in event.normalized() {
782                                if matches!(
783                                    chunk,
784                                    LlmResponseChunk::DecodeError { .. }
785                                        | LlmResponseChunk::StreamError { .. }
786                                ) {
787                                    valid = false;
788                                } else {
789                                    accumulator.push(chunk.clone());
790                                }
791                            }
792                        }
793                        Ok(Some((
794                            event,
795                            (
796                                stream,
797                                router,
798                                model,
799                                conversation,
800                                canonical_input,
801                                accumulator,
802                                native_responses,
803                                valid,
804                            ),
805                        )))
806                    },
807                )))
808            }
809        };
810        Ok(response)
811    }
812
813    /// Check both IDs from a response or stream event before recording either. The response's
814    /// `store` value overrides the request value; `false` skips only the response ID.
815    fn remember_response(
816        &self,
817        body: &Value,
818        model: &ModelId,
819        store: bool,
820        conversation: Option<&str>,
821    ) -> std::result::Result<(), LlmClientError> {
822        let mut owners = self.inner.state_owners.lock();
823        let response_id = body
824            .get("id")
825            .and_then(Value::as_str)
826            .filter(|_| body.get("store").and_then(Value::as_bool).unwrap_or(store));
827        let conversation = body.as_object().and_then(conversation_id).or(conversation);
828        if let Err(error) = owners.remember(response_id, conversation, model, None) {
829            if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
830                tracing::warn!(%error, "Responses state owner capacity reached; state was not retained");
831            } else {
832                tracing::error!(%error, "could not record Responses state owner");
833                return Err(error);
834            }
835        }
836        Ok(())
837    }
838
839    fn remember_canonical_response(
840        &self,
841        response: &AggLlmResponse,
842        model: &ModelId,
843        store: bool,
844        input: &CanonicalInput,
845    ) -> std::result::Result<(), LlmClientError> {
846        let response_id = response.id.as_deref().filter(|_| store);
847        if response_id.is_none() {
848            return Ok(());
849        }
850        let mut segment = input.messages.to_vec();
851        segment.extend(response.outputs.iter().map(|output| Message {
852            role: output.role,
853            content: output.content.clone(),
854        }));
855        let history = Arc::new(MessageHistory::new(
856            input.parent.clone(),
857            Arc::from(segment),
858        ));
859        let result =
860            self.inner
861                .state_owners
862                .lock()
863                .remember(response_id, None, model, Some(history));
864        if let Err(error) = result {
865            if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
866                tracing::warn!(%error, "cross-format Responses state capacity reached; history was not retained");
867                return Ok(());
868            }
869            tracing::error!(%error, "could not record cross-format Responses state");
870            return Err(error);
871        }
872        Ok(())
873    }
874
875    /// Prepare a completion candidate with its configured target prompt.
876    fn prepare_completion_request(&self, mut request: Request, target: &ModelId) -> Request {
877        let prompt = self.inner.target_prompts.get(target).map(String::as_str);
878        prepare_request_for_target(&mut request.llm_request, target, prompt);
879        request
880    }
881
882    /// Prepare a routing call, adding a target prompt only when it generates a candidate answer.
883    fn prepare_routing_request(&self, mut request: Request, target: &ModelId) -> Request {
884        let prompt = if self.inner.routing_answer_target.as_ref() == Some(target) {
885            self.inner.target_prompts.get(target).map(String::as_str)
886        } else {
887            None
888        };
889        prepare_request_for_target(&mut request.llm_request, target, prompt);
890        request
891    }
892}
893
894impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {
895    fn from_iter<I: IntoIterator<Item = (ModelId, Arc<dyn RoutedLlmClient>)>>(iter: I) -> Self {
896        Self::new(iter.into_iter().collect())
897    }
898}
899
900#[cfg(test)]
901mod tests {
902    use std::collections::BTreeMap;
903
904    use super::*;
905    use async_trait::async_trait;
906    use futures::StreamExt;
907    use http::StatusCode;
908    use switchyard_libsy::{Driver, RoutingOutcome};
909    use switchyard_protocol::{
910        ContentBlock, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Message, Role,
911        ToolResult, completion_text, text_request, text_response,
912    };
913    use wiremock::matchers::method;
914    use wiremock::{Mock, MockServer, ResponseTemplate};
915
916    use crate::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
917    use switchyard_protocol::Category;
918
919    struct CandidateAlgorithm {}
920
921    struct AnsweredAlgorithm {
922        model: ModelId,
923    }
924
925    #[async_trait]
926    impl Algorithm for CandidateAlgorithm {
927        fn name(&self) -> &str {
928            "candidate_test"
929        }
930
931        async fn route(
932            self: Arc<Self>,
933            driver: Driver,
934            request: Request,
935        ) -> Result<RoutingOutcome> {
936            let models = driver.models_for(&Category::Any);
937            let selected_model = models.first().cloned().ok_or(LibsyError::NoTargets)?;
938            Ok(RoutingOutcome::route_to(
939                selected_model,
940                models.iter().skip(1).cloned().collect(),
941                request,
942            ))
943        }
944    }
945
946    #[async_trait]
947    impl Algorithm for AnsweredAlgorithm {
948        fn name(&self) -> &str {
949            "answered_test"
950        }
951
952        async fn route(
953            self: Arc<Self>,
954            driver: Driver,
955            request: Request,
956        ) -> Result<RoutingOutcome> {
957            let response = driver
958                .call_model(request.clone(), vec![self.model.clone()])
959                .await?;
960            Ok(RoutingOutcome::answered(
961                self.model.clone(),
962                request,
963                response,
964            ))
965        }
966    }
967
968    #[derive(Clone, Copy)]
969    enum FirstOutcome {
970        ContextWindow,
971        ContentPolicy,
972        Unauthorized,
973        StreamSuccess,
974        MidStreamError,
975    }
976
977    struct CandidateClient {
978        calls: Mutex<Vec<ModelId>>,
979        requests: Mutex<Vec<Request>>,
980        first: FirstOutcome,
981    }
982
983    #[async_trait]
984    impl RoutedLlmClient for CandidateClient {
985        async fn call(&self, request: Request) -> std::result::Result<Response, LlmClientError> {
986            let model = request.model_id().unwrap_or_default();
987            self.calls.lock().push(model.clone());
988            self.requests.lock().push(request);
989            if model == "weak" {
990                return match self.first {
991                    FirstOutcome::ContextWindow => Err(LlmClientError::ContextWindowExceeded {
992                        model,
993                        message: "too long".to_string(),
994                    }),
995                    FirstOutcome::ContentPolicy => Err(LlmClientError::UpstreamHttp {
996                        status: StatusCode::BAD_REQUEST,
997                        body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(),
998                    }),
999                    FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {
1000                        status: StatusCode::UNAUTHORIZED,
1001                        body: "unauthorized".to_string(),
1002                    }),
1003                    FirstOutcome::StreamSuccess => Ok(stream_response(vec![
1004                        LlmResponseChunk::TextDelta {
1005                            index: 0,
1006                            text: "streamed".to_string(),
1007                        },
1008                        LlmResponseChunk::MessageStop {
1009                            reason: Some("stop".to_string()),
1010                        },
1011                    ])),
1012                    FirstOutcome::MidStreamError => Ok(stream_response(vec![
1013                        LlmResponseChunk::TextDelta {
1014                            index: 0,
1015                            text: "partial".to_string(),
1016                        },
1017                        LlmResponseChunk::StreamError {
1018                            message: "stream failed".to_string(),
1019                        },
1020                    ])),
1021                };
1022            }
1023            Ok(Response {
1024                llm_response: LlmResponse::Agg(text_response(Some(model.to_string()), model)),
1025                metadata: None,
1026                upstream_headers: http::HeaderMap::new(),
1027            })
1028        }
1029    }
1030
1031    fn stream_response(chunks: Vec<LlmResponseChunk>) -> Response {
1032        Response {
1033            llm_response: LlmResponse::Stream(
1034                futures::stream::iter(
1035                    chunks
1036                        .into_iter()
1037                        .map(|chunk| Ok(LlmResponseStreamEvent::from(chunk))),
1038                )
1039                .boxed(),
1040            ),
1041            metadata: None,
1042            upstream_headers: http::HeaderMap::new(),
1043        }
1044    }
1045
1046    fn request() -> Request {
1047        Request {
1048            llm_request: text_request(Some("auto".to_string()), "hello".to_string()),
1049            raw_request: None,
1050            metadata: None,
1051        }
1052    }
1053
1054    fn instruction_text(request: &Request) -> Vec<&str> {
1055        request
1056            .llm_request
1057            .instructions
1058            .iter()
1059            .flat_map(|instruction| &instruction.content)
1060            .filter_map(|block| match block {
1061                ContentBlock::Text { text } => Some(text.as_str()),
1062                _ => None,
1063            })
1064            .collect()
1065    }
1066
1067    async fn run_candidates(
1068        first: FirstOutcome,
1069    ) -> (Arc<CandidateClient>, Result<(ModelId, Response)>) {
1070        let client = Arc::new(CandidateClient {
1071            calls: Mutex::new(Vec::new()),
1072            requests: Mutex::new(Vec::new()),
1073            first,
1074        });
1075        let algorithm = Arc::new(CandidateAlgorithm {});
1076        let models = to_category_map(&["weak", "strong"]);
1077        let result = run(
1078            algorithm,
1079            ClientRouter::single(client.clone()),
1080            request(),
1081            models,
1082            None,
1083        )
1084        .await;
1085        (client, result)
1086    }
1087
1088    fn to_category_map(names: &[&str]) -> Arc<RuntimeModels> {
1089        Arc::new(RuntimeModels::new(
1090            [(
1091                Category::Any,
1092                names.iter().map(|name| ModelId::from(*name)).collect(),
1093            )]
1094            .into(),
1095        ))
1096    }
1097
1098    #[tokio::test]
1099    async fn answered_outcome_does_not_make_a_second_model_call() -> Result<()> {
1100        let client = Arc::new(CandidateClient {
1101            calls: Mutex::new(Vec::new()),
1102            requests: Mutex::new(Vec::new()),
1103            first: FirstOutcome::StreamSuccess,
1104        });
1105        let observations = Arc::new(Mutex::new(Vec::new()));
1106        let observed = Arc::clone(&observations);
1107        let observer: RunObserver = Arc::new(move |event| observed.lock().push(event));
1108
1109        let (selected, response) = run(
1110            Arc::new(AnsweredAlgorithm {
1111                model: "weak".into(),
1112            }),
1113            ClientRouter::single(client.clone()),
1114            request(),
1115            Arc::new(RuntimeModels::default()),
1116            Some(observer),
1117        )
1118        .await?;
1119
1120        assert_eq!(selected, "weak");
1121        assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));
1122        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
1123        let observations = observations.lock();
1124        assert!(matches!(observations[0], RunObservation::AnswerCall(_)));
1125        assert!(matches!(
1126            observations[1],
1127            RunObservation::Outcome(ref metadata) if metadata.algorithm == "answered_test"
1128        ));
1129        assert!(matches!(
1130            observations[2],
1131            RunObservation::RoutingOverhead(_)
1132        ));
1133        assert_eq!(observations.len(), 3);
1134        Ok(())
1135    }
1136
1137    #[test]
1138    fn answer_observation_keeps_call_order() {
1139        let pending = Some(Arc::new(Mutex::new(
1140            ["answer", "judge"]
1141                .map(|model| LlmCallObservation {
1142                    selected_model: model.into(),
1143                    is_success: true,
1144                    duration: std::time::Duration::ZERO,
1145                    usage: None,
1146                })
1147                .into(),
1148        )));
1149        let emitted = Arc::new(Mutex::new(Vec::new()));
1150        let captured = Arc::clone(&emitted);
1151        let observer: RunObserver = Arc::new(move |event| captured.lock().push(event));
1152        let answer = ModelId::from("answer");
1153
1154        emit_routing_observations(&Some(observer), &pending, Some(&answer));
1155
1156        assert!(matches!(
1157            &emitted.lock()[..],
1158            [RunObservation::AnswerCall(answer), RunObservation::LlmCall(judge)]
1159                if answer.selected_model == "answer" && judge.selected_model == "judge"
1160        ));
1161    }
1162
1163    #[tokio::test]
1164    async fn responses_state_tracking_uses_upstream_storage_and_format() -> Result<()> {
1165        for (responses, store, conversation) in [
1166            (true, true, None),
1167            (true, false, None),
1168            (true, false, Some(json!("conv_plain"))),
1169            (true, false, Some(json!({"id": "conv_object"}))),
1170            (false, true, None),
1171        ] {
1172            for stream in [false, true] {
1173                let server = MockServer::start().await;
1174                Mock::given(method("POST"))
1175                    .respond_with(move |request: &wiremock::Request| {
1176                        let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
1177                        let response = if responses {
1178                            json!({
1179                                "id": "resp_seed", "object": "response", "model": "weak",
1180                                "status": "completed", "output": [], "store": body["store"],
1181                                "conversation": body["conversation"],
1182                            })
1183                        } else if stream {
1184                            json!({"id": "resp_seed", "model": "weak", "choices": [{"index": 0, "delta": {"content": "ok"}, "finish_reason": "stop"}]})
1185                        } else {
1186                            json!({"id": "resp_seed", "model": "weak", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}]})
1187                        };
1188                        if stream {
1189                            let event = if responses { json!({"type": "response.completed", "response": response}) } else { response };
1190                            ResponseTemplate::new(200)
1191                                .insert_header("content-type", "text/event-stream")
1192                                .set_body_string(if responses {
1193                                    format!("event: response.completed\ndata: {event}\n\n")
1194                                } else {
1195                                    format!("data: {event}\n\ndata: [DONE]\n\n")
1196                                })
1197                        } else {
1198                            ResponseTemplate::new(200).set_body_json(response)
1199                        }
1200                    })
1201                    .mount(&server).await;
1202                let mut clients = HashMap::new();
1203                for model in ["weak", "strong"] {
1204                    let config = HttpBackendConfig {
1205                        base_url: server.uri(),
1206                        api_key: None,
1207                        forward_auth: false,
1208                        extra_headers: BTreeMap::new(),
1209                        extra_body: BTreeMap::from([("store".to_string(), json!(store))]),
1210                        reasoning_effort: None,
1211                        max_retries: 0,
1212                        timeout: None,
1213                    };
1214                    let backend = if responses {
1215                        Backend::OpenAiResponses(config)
1216                    } else {
1217                        Backend::OpenAiChat(config)
1218                    };
1219                    let client: Arc<dyn RoutedLlmClient> = Arc::new(
1220                        TranslatingLlmClient::new(&[ModelConfig::new(model, backend, None)])
1221                            .map_err(|error| LibsyError::external("building test client", error))?,
1222                    );
1223                    clients.insert(ModelId::from(model), client);
1224                }
1225                let clients = ClientRouter::new(clients);
1226                let mut seed = request();
1227                seed.llm_request.stream = stream;
1228                if let Some(conversation) = &conversation {
1229                    seed.llm_request
1230                        .extensions
1231                        .fields
1232                        .insert("conversation".to_string(), conversation.clone());
1233                }
1234                let (_, response) = run(
1235                    Arc::new(switchyard_libsy::Passthrough),
1236                    clients.clone(),
1237                    seed,
1238                    to_category_map(&["weak"]),
1239                    None,
1240                )
1241                .await?;
1242                response
1243                    .llm_response
1244                    .into_agg()
1245                    .await
1246                    .map_err(|error| LibsyError::client_call("weak", error))?;
1247                let mut continuations = vec![(
1248                    "previous_response_id",
1249                    json!("resp_seed"),
1250                    responses && store,
1251                )];
1252                if let Some(conversation) = conversation.clone() {
1253                    continuations.push(("conversation", conversation, true));
1254                }
1255                for (field, id, pinned) in continuations {
1256                    let mut follow = request();
1257                    follow
1258                        .llm_request
1259                        .extensions
1260                        .fields
1261                        .insert(field.to_string(), id);
1262                    let outcome = decide(
1263                        Arc::new(switchyard_libsy::Passthrough),
1264                        clients.clone(),
1265                        follow,
1266                        to_category_map(&["strong"]),
1267                    )
1268                    .await?;
1269                    assert_eq!(
1270                        outcome.selected_model_id()?.as_str(),
1271                        if pinned { "weak" } else { "strong" },
1272                        "responses={responses}, store={store}, stream={stream}, field={field}"
1273                    );
1274                }
1275                let requests = server.received_requests().await.expect("request recording");
1276                assert_eq!(requests.len(), 1);
1277                assert_eq!(
1278                    serde_json::from_slice::<Value>(&requests[0].body).expect("request JSON")["store"],
1279                    store
1280                );
1281            }
1282        }
1283        Ok(())
1284    }
1285
1286    #[tokio::test]
1287    async fn responses_stored_tool_continuation_materializes_for_anthropic() -> Result<()> {
1288        let server = MockServer::start().await;
1289        Mock::given(method("POST"))
1290            .respond_with(|request: &wiremock::Request| {
1291                let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
1292                let blocks = body["messages"]
1293                    .as_array()
1294                    .into_iter()
1295                    .flatten()
1296                    .filter_map(|message| message["content"].as_array())
1297                    .flatten()
1298                    .collect::<Vec<_>>();
1299                let has_result = blocks.iter().any(|block| block["type"] == "tool_result");
1300                let has_call = blocks.iter().any(|block| block["type"] == "tool_use");
1301                let has_thanks = blocks
1302                    .iter()
1303                    .any(|block| block["type"] == "text" && block["text"] == "thanks")
1304                    || body["messages"].as_array().is_some_and(|messages| {
1305                        messages
1306                            .iter()
1307                            .any(|message| message["content"] == "thanks")
1308                    });
1309                if has_result && !has_call {
1310                    return ResponseTemplate::new(400).set_body_json(json!({
1311                        "type": "error",
1312                        "error": {
1313                            "type": "invalid_request_error",
1314                            "message": "tool_result has no corresponding tool_use"
1315                        }
1316                    }));
1317                }
1318                let response = if has_thanks {
1319                    json!({
1320                        "id": "msg_third", "type": "message", "role": "assistant",
1321                        "model": "weak", "content": [{"type": "text", "text": "done"}],
1322                        "stop_reason": "end_turn", "usage": {"input_tokens": 6, "output_tokens": 1}
1323                    })
1324                } else if has_result {
1325                    json!({
1326                        "id": "msg_follow", "type": "message", "role": "assistant",
1327                        "model": "weak", "content": [{"type": "text", "text": "sunny"}],
1328                        "stop_reason": "end_turn", "usage": {"input_tokens": 4, "output_tokens": 1}
1329                    })
1330                } else {
1331                    json!({
1332                        "id": "msg_seed", "type": "message", "role": "assistant",
1333                        "model": "weak",
1334                        "content": [{
1335                            "type": "tool_use", "id": "toolu_weather", "name": "get_weather",
1336                            "input": {"city": "Paris"}
1337                        }],
1338                        "stop_reason": "tool_use", "usage": {"input_tokens": 3, "output_tokens": 1}
1339                    })
1340                };
1341                ResponseTemplate::new(200).set_body_json(response)
1342            })
1343            .mount(&server)
1344            .await;
1345
1346        let client: Arc<dyn RoutedLlmClient> = Arc::new(
1347            TranslatingLlmClient::new(&[ModelConfig::new(
1348                "weak",
1349                Backend::Anthropic(HttpBackendConfig {
1350                    base_url: server.uri(),
1351                    api_key: None,
1352                    forward_auth: false,
1353                    extra_headers: BTreeMap::new(),
1354                    extra_body: BTreeMap::new(),
1355                    reasoning_effort: None,
1356                    max_retries: 0,
1357                    timeout: None,
1358                }),
1359                None,
1360            )])
1361            .map_err(|error| LibsyError::external("building test client", error))?,
1362        );
1363        let clients = ClientRouter::new(HashMap::from([(ModelId::from("weak"), client)]));
1364        let models = to_category_map(&["weak"]);
1365
1366        let seed = Request {
1367            llm_request: switchyard_translation::decode_request(
1368                WireFormat::OpenAiResponses,
1369                &json!({
1370                    "model": "route",
1371                    "input": "Call get_weather for Paris",
1372                    "tools": [{
1373                        "type": "function", "name": "get_weather",
1374                        "parameters": {"type": "object"}
1375                    }],
1376                    "store": true
1377                }),
1378            )
1379            .map_err(|error| LibsyError::external("decoding seed request", error))?,
1380            raw_request: None,
1381            metadata: None,
1382        };
1383        let (_, response) = run(
1384            Arc::new(switchyard_libsy::Passthrough),
1385            clients.clone(),
1386            seed,
1387            models.clone(),
1388            None,
1389        )
1390        .await?;
1391        assert_eq!(
1392            response
1393                .llm_response
1394                .as_agg()
1395                .and_then(|agg| agg.id.as_deref()),
1396            Some("msg_seed")
1397        );
1398
1399        let follow = Request {
1400            llm_request: switchyard_translation::decode_request(
1401                WireFormat::OpenAiResponses,
1402                &json!({
1403                    "model": "route",
1404                    "previous_response_id": "msg_seed",
1405                    "input": [{
1406                        "type": "function_call_output",
1407                        "call_id": "toolu_weather",
1408                        "output": "Paris is sunny"
1409                    }],
1410                    "tools": [{
1411                        "type": "function", "name": "get_weather",
1412                        "parameters": {"type": "object"}
1413                    }],
1414                    "store": true
1415                }),
1416            )
1417            .map_err(|error| LibsyError::external("decoding continuation request", error))?,
1418            raw_request: None,
1419            metadata: None,
1420        };
1421        let (_, response) = run(
1422            Arc::new(switchyard_libsy::Passthrough),
1423            clients.clone(),
1424            follow,
1425            models.clone(),
1426            None,
1427        )
1428        .await?;
1429        assert_eq!(
1430            completion_text(
1431                response
1432                    .llm_response
1433                    .as_agg()
1434                    .expect("buffered Anthropic response"),
1435            ),
1436            "sunny"
1437        );
1438
1439        let third = Request {
1440            llm_request: switchyard_translation::decode_request(
1441                WireFormat::OpenAiResponses,
1442                &json!({
1443                    "model": "route",
1444                    "previous_response_id": "msg_follow",
1445                    "input": "thanks",
1446                    "store": true
1447                }),
1448            )
1449            .map_err(|error| LibsyError::external("decoding chained request", error))?,
1450            raw_request: None,
1451            metadata: None,
1452        };
1453        let history = clients
1454            .stored_state_owner(&third)
1455            .and_then(|owner| owner.history)
1456            .expect("chained canonical history");
1457        assert_eq!(history.len, 4);
1458        assert_eq!(history.segment.len(), 2);
1459        assert!(history.parent.is_some());
1460        let (_, response) = run(
1461            Arc::new(switchyard_libsy::Passthrough),
1462            clients,
1463            third,
1464            models,
1465            None,
1466        )
1467        .await?;
1468        assert_eq!(
1469            completion_text(
1470                response
1471                    .llm_response
1472                    .as_agg()
1473                    .expect("buffered Anthropic response"),
1474            ),
1475            "done"
1476        );
1477
1478        let requests = server.received_requests().await.expect("request recording");
1479        assert_eq!(requests.len(), 3);
1480        let follow: Value = serde_json::from_slice(&requests[1].body)
1481            .map_err(|error| LibsyError::external("decoding captured request", error))?;
1482        assert_eq!(follow["messages"][1]["content"][0]["type"], "tool_use");
1483        assert_eq!(follow["messages"][2]["content"][0]["type"], "tool_result");
1484        let third: Value = serde_json::from_slice(&requests[2].body)
1485            .map_err(|error| LibsyError::external("decoding captured request", error))?;
1486        assert_eq!(third["messages"][1]["content"][0]["type"], "tool_use");
1487        assert_eq!(third["messages"][2]["content"][0]["type"], "tool_result");
1488        assert_eq!(third["messages"][3]["content"], "sunny");
1489        assert_eq!(third["messages"][4]["content"][0]["text"], "thanks");
1490        Ok(())
1491    }
1492
1493    #[tokio::test]
1494    async fn streamed_cross_format_state_is_recorded_only_after_completion() -> Result<()> {
1495        let client = Arc::new(CandidateClient {
1496            calls: Mutex::new(Vec::new()),
1497            requests: Mutex::new(Vec::new()),
1498            first: FirstOutcome::StreamSuccess,
1499        });
1500        let clients = ClientRouter::new(HashMap::from([(
1501            ModelId::from("weak"),
1502            client as Arc<dyn RoutedLlmClient>,
1503        )]));
1504        let mut seed = request();
1505        seed.llm_request.messages = vec![Message::text(Role::User, "inspect")];
1506        seed.llm_request
1507            .preservation
1508            .requests
1509            .insert(WireFormat::OpenAiResponses.into(), json!({}));
1510        let mut response = stream_response(vec![
1511            LlmResponseChunk::MessageStart {
1512                id: Some("msg_stream".to_string()),
1513                model: Some("weak".to_string()),
1514            },
1515            LlmResponseChunk::ToolCallDelta {
1516                index: 0,
1517                id: Some("toolu_stream".to_string()),
1518                name: Some("lookup".to_string()),
1519                arguments_delta: Some("{}".to_string()),
1520            },
1521            LlmResponseChunk::MessageStop {
1522                reason: Some("tool_use".to_string()),
1523            },
1524        ]);
1525        response.set_served_model(&ModelId::from("weak"));
1526        let response = clients.remember_state_owner(&seed, response)?;
1527
1528        let mut follow = request();
1529        follow
1530            .llm_request
1531            .extensions
1532            .fields
1533            .insert("previous_response_id".to_string(), json!("msg_stream"));
1534        follow.llm_request.messages = vec![Message {
1535            role: Role::Tool,
1536            content: vec![ContentBlock::ToolResult(ToolResult {
1537                tool_call_id: "toolu_stream".to_string(),
1538                content: vec![ContentBlock::Text {
1539                    text: "ready".to_string(),
1540                }],
1541                is_error: None,
1542            })],
1543        }];
1544        assert!(clients.stored_state_owner(&follow).is_none());
1545        response
1546            .llm_response
1547            .into_agg()
1548            .await
1549            .map_err(|error| LibsyError::client_call("weak", error))?;
1550
1551        let outcome = continue_on(
1552            clients
1553                .stored_state_owner(&follow)
1554                .expect("completed stream state"),
1555            "passthrough",
1556            follow,
1557        );
1558        assert_eq!(outcome.request.llm_request.messages.len(), 3);
1559        assert!(matches!(
1560            outcome.request.llm_request.messages[1].content.as_slice(),
1561            [ContentBlock::ToolCall(call)] if call.id == "toolu_stream"
1562        ));
1563        assert!(matches!(
1564            outcome.request.llm_request.messages[2].content.as_slice(),
1565            [ContentBlock::ToolResult(result)] if result.tool_call_id == "toolu_stream"
1566        ));
1567        Ok(())
1568    }
1569
1570    #[test]
1571    fn cross_format_response_with_store_false_is_not_retained() -> Result<()> {
1572        let client = Arc::new(CandidateClient {
1573            calls: Mutex::new(Vec::new()),
1574            requests: Mutex::new(Vec::new()),
1575            first: FirstOutcome::StreamSuccess,
1576        });
1577        let clients = ClientRouter::new(HashMap::from([(
1578            ModelId::from("weak"),
1579            client as Arc<dyn RoutedLlmClient>,
1580        )]));
1581        let mut seed = request();
1582        seed.llm_request
1583            .extensions
1584            .fields
1585            .insert("store".to_string(), json!(false));
1586        seed.llm_request
1587            .preservation
1588            .requests
1589            .insert(WireFormat::OpenAiResponses.into(), json!({}));
1590        let mut agg = text_response(Some("weak".to_string()), "done".to_string());
1591        agg.id = Some("msg_not_stored".to_string());
1592        let mut response = Response {
1593            llm_response: LlmResponse::Agg(agg),
1594            metadata: None,
1595            upstream_headers: http::HeaderMap::new(),
1596        };
1597        response.set_served_model(&ModelId::from("weak"));
1598        clients.remember_state_owner(&seed, response)?;
1599
1600        let mut follow = request();
1601        follow
1602            .llm_request
1603            .extensions
1604            .fields
1605            .insert("previous_response_id".to_string(), json!("msg_not_stored"));
1606        assert!(clients.stored_state_owner(&follow).is_none());
1607        Ok(())
1608    }
1609
1610    #[tokio::test]
1611    async fn state_capacity_does_not_fail_completed_responses() -> Result<()> {
1612        let client = Arc::new(CandidateClient {
1613            calls: Mutex::new(Vec::new()),
1614            requests: Mutex::new(Vec::new()),
1615            first: FirstOutcome::StreamSuccess,
1616        });
1617        let clients = ClientRouter::new(HashMap::from([(
1618            ModelId::from("weak"),
1619            client as Arc<dyn RoutedLlmClient>,
1620        )]));
1621        let model = ModelId::from("weak");
1622        clients.inner.state_owners.lock().by_id = (0..MAX_STATE_OWNERS)
1623            .map(|id| {
1624                (
1625                    format!("existing_{id}"),
1626                    StoredState {
1627                        model: model.clone(),
1628                        history: None,
1629                    },
1630                )
1631            })
1632            .collect();
1633
1634        let mut request = request();
1635        request
1636            .llm_request
1637            .preservation
1638            .requests
1639            .insert(WireFormat::OpenAiResponses.into(), json!({}));
1640        let mut agg = text_response(Some("weak".to_string()), "done".to_string());
1641        agg.id = Some("overflow_buffered".to_string());
1642        let mut buffered = Response {
1643            llm_response: LlmResponse::Agg(agg),
1644            metadata: None,
1645            upstream_headers: http::HeaderMap::new(),
1646        };
1647        buffered.set_served_model(&model);
1648        clients.remember_state_owner(&request, buffered)?;
1649
1650        let mut streamed = stream_response(vec![
1651            LlmResponseChunk::MessageStart {
1652                id: Some("overflow_streamed".to_string()),
1653                model: Some("weak".to_string()),
1654            },
1655            LlmResponseChunk::TextDelta {
1656                index: 0,
1657                text: "done".to_string(),
1658            },
1659            LlmResponseChunk::MessageStop {
1660                reason: Some("stop".to_string()),
1661            },
1662        ]);
1663        streamed.set_served_model(&model);
1664        clients
1665            .remember_state_owner(&request, streamed)?
1666            .llm_response
1667            .into_agg()
1668            .await
1669            .map_err(|error| LibsyError::client_call("weak", error))?;
1670
1671        assert_eq!(
1672            clients.inner.state_owners.lock().by_id.len(),
1673            MAX_STATE_OWNERS
1674        );
1675        Ok(())
1676    }
1677
1678    #[test]
1679    fn state_owners_preserve_existing_records_at_capacity()
1680    -> std::result::Result<(), LlmClientError> {
1681        let mut owners = StateOwners::default();
1682        let model = ModelId::from("model/a");
1683        for id in 0..MAX_STATE_OWNERS - 1 {
1684            owners.remember(Some(&format!("resp_{id}")), None, &model, None)?;
1685        }
1686        assert!(matches!(
1687            owners.remember(Some("new_response"), Some("new_conversation"), &model, None,),
1688            Err(LlmClientError::ResponseStateLimitExceeded {
1689                limit: MAX_STATE_OWNERS
1690            })
1691        ));
1692        assert!(owners.owner("new_response").is_none());
1693        assert!(owners.owner("new_conversation").is_none());
1694        owners.remember(Some("last_id"), Some("last_id"), &model, None)?;
1695        owners.remember(Some("last_id"), Some("resp_0"), &model, None)?;
1696        assert!(matches!(
1697            owners.remember(Some("over_limit"), None, &model, None),
1698            Err(LlmClientError::ResponseStateLimitExceeded { .. })
1699        ));
1700        assert!(matches!(
1701            owners.remember(Some("resp_0"), None, &ModelId::from("model/b"), None,),
1702            Err(LlmClientError::ResponseStateConflict)
1703        ));
1704        assert_eq!(
1705            owners.owner("resp_0").map(|state| &state.model),
1706            Some(&model)
1707        );
1708        assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
1709        Ok(())
1710    }
1711
1712    #[test]
1713    fn concurrent_state_owners_do_not_exceed_capacity() {
1714        let model = ModelId::from("model/a");
1715        let owners = Arc::new(Mutex::new(StateOwners {
1716            by_id: (0..MAX_STATE_OWNERS - 16)
1717                .map(|id| {
1718                    (
1719                        format!("existing_{id}"),
1720                        StoredState {
1721                            model: model.clone(),
1722                            history: None,
1723                        },
1724                    )
1725                })
1726                .collect(),
1727        }));
1728        let barrier = Arc::new(std::sync::Barrier::new(32));
1729        let accepted = std::thread::scope(|scope| {
1730            let handles = (0..32)
1731                .map(|id| {
1732                    let owners = owners.clone();
1733                    let barrier = barrier.clone();
1734                    let model = model.clone();
1735                    scope.spawn(move || {
1736                        barrier.wait();
1737                        let result = owners.lock().remember(
1738                            Some(&format!("resp_{id}")),
1739                            Some(&format!("conv_{id}")),
1740                            &model,
1741                            None,
1742                        );
1743                        match result {
1744                            Ok(()) => Some(id),
1745                            Err(LlmClientError::ResponseStateLimitExceeded { .. }) => None,
1746                            other => panic!("unexpected state tracking result: {other:?}"),
1747                        }
1748                    })
1749                })
1750                .collect::<Vec<_>>();
1751            handles
1752                .into_iter()
1753                .filter_map(|handle| handle.join().expect("request thread panicked"))
1754                .collect::<Vec<_>>()
1755        });
1756        let owners = owners.lock();
1757        assert_eq!(accepted.len(), 8);
1758        assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
1759        assert_eq!(
1760            owners.owner("existing_0").map(|state| &state.model),
1761            Some(&model)
1762        );
1763        for id in accepted {
1764            assert_eq!(
1765                owners
1766                    .owner(&format!("resp_{id}"))
1767                    .map(|state| &state.model),
1768                Some(&model)
1769            );
1770            assert_eq!(
1771                owners
1772                    .owner(&format!("conv_{id}"))
1773                    .map(|state| &state.model),
1774                Some(&model)
1775            );
1776        }
1777    }
1778
1779    #[tokio::test]
1780    async fn each_fallback_candidate_receives_only_its_own_prompt() -> Result<()> {
1781        let client = Arc::new(CandidateClient {
1782            calls: Mutex::new(Vec::new()),
1783            requests: Mutex::new(Vec::new()),
1784            first: FirstOutcome::ContextWindow,
1785        });
1786        let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
1787        let clients = ClientRouter::new_with_target_prompts(
1788            HashMap::from([
1789                (ModelId::from("weak"), Arc::clone(&routed_client)),
1790                (ModelId::from("strong"), routed_client),
1791            ]),
1792            HashMap::from([
1793                ("weak".into(), "weak prompt".to_string()),
1794                ("strong".into(), "strong prompt".to_string()),
1795            ]),
1796            None,
1797        );
1798
1799        run(
1800            Arc::new(CandidateAlgorithm {}),
1801            clients,
1802            request(),
1803            to_category_map(&["weak", "strong"]),
1804            None,
1805        )
1806        .await?;
1807
1808        let calls = client.requests.lock();
1809        assert_eq!(calls.len(), 2);
1810        assert_eq!(instruction_text(&calls[0]), ["weak prompt"]);
1811        assert_eq!(instruction_text(&calls[1]), ["strong prompt"]);
1812        Ok(())
1813    }
1814
1815    #[tokio::test]
1816    async fn decision_prompts_a_routing_response_target() -> Result<()> {
1817        let client = Arc::new(CandidateClient {
1818            calls: Mutex::new(Vec::new()),
1819            requests: Mutex::new(Vec::new()),
1820            first: FirstOutcome::StreamSuccess,
1821        });
1822        let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
1823        let clients = ClientRouter::new_with_target_prompts(
1824            HashMap::from([(ModelId::from("answer"), routed_client)]),
1825            HashMap::from([("answer".into(), "answer prompt".to_string())]),
1826            Some("answer".into()),
1827        );
1828
1829        decide(
1830            Arc::new(AnsweredAlgorithm {
1831                model: "answer".into(),
1832            }),
1833            clients,
1834            request(),
1835            Arc::new(RuntimeModels::default()),
1836        )
1837        .await?;
1838
1839        let calls = client.requests.lock();
1840        assert_eq!(calls.len(), 1);
1841        assert_eq!(instruction_text(&calls[0]), ["answer prompt"]);
1842        Ok(())
1843    }
1844
1845    #[tokio::test]
1846    async fn decision_prepares_only_the_selected_request() -> Result<()> {
1847        let client = Arc::new(CandidateClient {
1848            calls: Mutex::new(Vec::new()),
1849            requests: Mutex::new(Vec::new()),
1850            first: FirstOutcome::StreamSuccess,
1851        });
1852        let routed_client: Arc<dyn RoutedLlmClient> = client;
1853        let clients = ClientRouter::new_with_target_prompts(
1854            HashMap::from([
1855                (ModelId::from("weak"), Arc::clone(&routed_client)),
1856                (ModelId::from("strong"), routed_client),
1857            ]),
1858            HashMap::from([
1859                (ModelId::from("weak"), "weak prompt".to_string()),
1860                (ModelId::from("strong"), "strong prompt".to_string()),
1861            ]),
1862            None,
1863        );
1864
1865        let outcome = decide(
1866            Arc::new(CandidateAlgorithm {}),
1867            clients,
1868            request(),
1869            to_category_map(&["weak", "strong"]),
1870        )
1871        .await?;
1872
1873        assert_eq!(
1874            outcome.selected_model_ids,
1875            [ModelId::from("weak"), ModelId::from("strong")]
1876        );
1877        assert_eq!(instruction_text(&outcome.request), ["weak prompt"]);
1878        Ok(())
1879    }
1880
1881    #[test]
1882    fn fallback_only_accepts_context_policy_and_unavailable_failures() {
1883        let error = |source| LibsyError::client_call("target", source);
1884        assert_eq!(
1885            fallback_reason(&error(LlmClientError::ContextWindowExceeded {
1886                model: "target".into(),
1887                message: "too long".to_string(),
1888            })),
1889            Some(RoutingFallbackReason::ContextWindow)
1890        );
1891        for status in [
1892            StatusCode::FORBIDDEN,
1893            StatusCode::REQUEST_TIMEOUT,
1894            StatusCode::TOO_MANY_REQUESTS,
1895            StatusCode::INTERNAL_SERVER_ERROR,
1896            StatusCode::from_u16(599).expect("599 is a valid status"),
1897        ] {
1898            assert_eq!(
1899                fallback_reason(&error(LlmClientError::UpstreamHttp {
1900                    status,
1901                    body: "failed".to_string(),
1902                })),
1903                Some(RoutingFallbackReason::Unavailable)
1904            );
1905        }
1906        for (body, expected) in [
1907            (
1908                r#"{"error":{"code":"content_policy_violation"}}"#,
1909                Some(RoutingFallbackReason::Unavailable),
1910            ),
1911            (
1912                r#"{"error":{"code":"invalid_request_error"},"metadata":"content_policy_violation"}"#,
1913                None,
1914            ),
1915            (r#"{"error":{"message":"content_policy_violation"}}"#, None),
1916            ("content_policy_violation", None),
1917        ] {
1918            assert_eq!(
1919                fallback_reason(&error(LlmClientError::UpstreamHttp {
1920                    status: StatusCode::BAD_REQUEST,
1921                    body: body.to_string(),
1922                })),
1923                expected,
1924                "{body}"
1925            );
1926        }
1927        for status in [
1928            StatusCode::BAD_REQUEST,
1929            StatusCode::UNAUTHORIZED,
1930            StatusCode::NOT_FOUND,
1931            StatusCode::CONFLICT,
1932            StatusCode::from_u16(499).expect("499 is a valid status"),
1933            StatusCode::from_u16(600).expect("600 is a valid status"),
1934        ] {
1935            assert_eq!(
1936                fallback_reason(&error(LlmClientError::UpstreamHttp {
1937                    status,
1938                    body: "failed".to_string(),
1939                })),
1940                None
1941            );
1942        }
1943    }
1944
1945    #[tokio::test]
1946    async fn candidate_failures_follow_the_fallback_policy() -> Result<()> {
1947        // Context overflow is retryable across candidates.
1948        let (client, result) = run_candidates(FirstOutcome::ContextWindow).await;
1949        let (_, response) = result?;
1950        assert_eq!(
1951            &*client.calls.lock(),
1952            &[ModelId::from("weak"), "strong".into()]
1953        );
1954        assert_eq!(
1955            response
1956                .llm_response
1957                .as_agg()
1958                .map(|response| response.model.as_deref()),
1959            Some(Some("strong"))
1960        );
1961        assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
1962
1963        // A provider-specific policy denial also permits another candidate.
1964        let (client, result) = run_candidates(FirstOutcome::ContentPolicy).await;
1965        let (_, response) = result?;
1966        assert_eq!(
1967            &*client.calls.lock(),
1968            &[ModelId::from("weak"), "strong".into()]
1969        );
1970        assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
1971
1972        // Authentication failure is not retryable, so the second candidate is untouched.
1973        let (client, result) = run_candidates(FirstOutcome::Unauthorized).await;
1974        assert!(matches!(
1975            result,
1976            Err(LibsyError::ClientCall {
1977                source: LlmClientError::UpstreamHttp {
1978                    status: StatusCode::UNAUTHORIZED,
1979                    ..
1980                },
1981                ..
1982            })
1983        ));
1984        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
1985        Ok(())
1986    }
1987
1988    #[tokio::test]
1989    async fn retry_budget_is_exhausted_before_falling_through() -> Result<()> {
1990        let server = MockServer::start().await;
1991        let calls = Arc::new(Mutex::new(Vec::new()));
1992        let observed_calls = Arc::clone(&calls);
1993        Mock::given(method("POST"))
1994            .respond_with(move |request: &wiremock::Request| {
1995                let body: serde_json::Value =
1996                    serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
1997                let model = body["model"].as_str().unwrap_or_default().to_string();
1998                observed_calls.lock().push(model.clone());
1999                if model == "weak" {
2000                    ResponseTemplate::new(503)
2001                        .insert_header("retry-after", "0")
2002                        .set_body_string("unavailable")
2003                } else {
2004                    ResponseTemplate::new(200).set_body_json(serde_json::json!({
2005                        "id": "answer",
2006                        "model": "strong",
2007                        "choices": [{
2008                            "index": 0,
2009                            "message": {"role": "assistant", "content": "ok"},
2010                            "finish_reason": "stop"
2011                        }],
2012                        "usage": {}
2013                    }))
2014                }
2015            })
2016            .mount(&server)
2017            .await;
2018
2019        let backend = || {
2020            Backend::OpenAiChat(HttpBackendConfig {
2021                base_url: format!("{}/v1", server.uri()),
2022                api_key: None,
2023                forward_auth: false,
2024                extra_headers: BTreeMap::new(),
2025                extra_body: BTreeMap::new(),
2026                reasoning_effort: None,
2027                max_retries: 2,
2028                timeout: None,
2029            })
2030        };
2031        let client = Arc::new(
2032            TranslatingLlmClient::new(&[
2033                ModelConfig::new("weak", backend(), None),
2034                ModelConfig::new("strong", backend(), None),
2035            ])
2036            .map_err(|error| LibsyError::external("building test client", error))?,
2037        );
2038        let algorithm = Arc::new(CandidateAlgorithm {});
2039        run(
2040            algorithm,
2041            ClientRouter::single(client),
2042            request(),
2043            to_category_map(&["weak", "strong"]),
2044            None,
2045        )
2046        .await?;
2047
2048        assert_eq!(&*calls.lock(), &["weak", "weak", "weak", "strong"]);
2049        Ok(())
2050    }
2051
2052    #[tokio::test]
2053    async fn streams_are_outside_the_candidate_fallback_boundary() -> Result<()> {
2054        // Receiving a stream handle is a successful call and ends candidate selection.
2055        let (client, result) = run_candidates(FirstOutcome::StreamSuccess).await;
2056        let (_, response) = result?;
2057        assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));
2058        let aggregate = response
2059            .llm_response
2060            .into_agg()
2061            .await
2062            .map_err(|source| LibsyError::client_call("weak", source))?;
2063        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2064        assert_eq!(completion_text(&aggregate), "streamed");
2065
2066        // A later in-stream failure surfaces during aggregation without trying another model.
2067        let (client, result) = run_candidates(FirstOutcome::MidStreamError).await;
2068        let (_, response) = result?;
2069        let error = response
2070            .llm_response
2071            .into_agg()
2072            .await
2073            .expect_err("the stream should fail during aggregation");
2074        assert!(error.to_string().contains("stream failed"));
2075        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2076        Ok(())
2077    }
2078
2079    // Verbatim shape of an SGLang admission rejection on a streaming session:
2080    // HTTP 200, and the overflow arrives as the first SSE event.
2081    const OVERFLOW_SSE: &str = "data: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.\",\"object\":\"error\",\"param\":null,\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n";
2082
2083    // A normal one-turn streamed answer, as the second candidate serves it.
2084    const STRONG_SSE: &str = "data: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
2085
2086    /// Runs a streaming request through the weak→strong candidate loop, serving
2087    /// `weak_sse` from the first candidate and [`STRONG_SSE`] from the second.
2088    /// The server is returned so response streams stay readable after the call.
2089    async fn run_streaming_candidates(
2090        weak_sse: &'static str,
2091    ) -> (
2092        MockServer,
2093        Arc<Mutex<Vec<String>>>,
2094        Result<(ModelId, Response)>,
2095    ) {
2096        let server = MockServer::start().await;
2097        let calls = Arc::new(Mutex::new(Vec::new()));
2098        let observed_calls = Arc::clone(&calls);
2099        Mock::given(method("POST"))
2100            .respond_with(move |request: &wiremock::Request| {
2101                let body: serde_json::Value =
2102                    serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
2103                let model = body["model"].as_str().unwrap_or_default().to_string();
2104                observed_calls.lock().push(model.clone());
2105                let sse = if model == "weak" {
2106                    weak_sse
2107                } else {
2108                    STRONG_SSE
2109                };
2110                ResponseTemplate::new(200)
2111                    .insert_header("content-type", "text/event-stream")
2112                    .set_body_string(sse)
2113            })
2114            .mount(&server)
2115            .await;
2116
2117        let backend = || {
2118            Backend::OpenAiChat(HttpBackendConfig {
2119                base_url: format!("{}/v1", server.uri()),
2120                api_key: None,
2121                forward_auth: false,
2122                extra_headers: BTreeMap::new(),
2123                extra_body: BTreeMap::new(),
2124                reasoning_effort: None,
2125                max_retries: 0,
2126                timeout: None,
2127            })
2128        };
2129        let client = Arc::new(
2130            TranslatingLlmClient::new(&[
2131                ModelConfig::new("weak", backend(), None),
2132                ModelConfig::new("strong", backend(), None),
2133            ])
2134            .expect("building test client"),
2135        );
2136        let algorithm = Arc::new(CandidateAlgorithm {});
2137        let mut llm_request = text_request(Some("auto".to_string()), "hello".to_string());
2138        llm_request.stream = true;
2139        let request = Request {
2140            llm_request,
2141            raw_request: None,
2142            metadata: None,
2143        };
2144        let result = run(
2145            algorithm,
2146            ClientRouter::single(client),
2147            request,
2148            to_category_map(&["weak", "strong"]),
2149            None,
2150        )
2151        .await;
2152        (server, calls, result)
2153    }
2154
2155    #[tokio::test]
2156    async fn first_stream_event_overflow_falls_back_to_the_next_candidate() -> Result<()> {
2157        // Nothing has reached the caller when the first event arrives, so an overflow
2158        // there falls through to the next candidate exactly as its buffered twin does.
2159        let (_server, calls, result) = run_streaming_candidates(OVERFLOW_SSE).await;
2160        let (_, response) = result?;
2161        assert_eq!(&*calls.lock(), &["weak", "strong"]);
2162        assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2163        let aggregate = response
2164            .llm_response
2165            .into_agg()
2166            .await
2167            .map_err(|source| LibsyError::client_call("strong", source))?;
2168        assert_eq!(completion_text(&aggregate), "ok");
2169
2170        // Only a classified overflow re-routes; any other in-band error event still
2171        // streams through to the consumer without trying another candidate.
2172        let (_server, calls, result) = run_streaming_candidates(
2173            "data: {\"error\":{\"code\":500,\"message\":\"engine exploded\",\"object\":\"error\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
2174        )
2175        .await;
2176        let (_, response) = result?;
2177        assert_eq!(&*calls.lock(), &["weak"]);
2178        let error = response
2179            .llm_response
2180            .into_agg()
2181            .await
2182            .expect_err("the in-band error should surface to the consumer");
2183        assert!(error.to_string().contains("engine exploded"));
2184
2185        // Once content has streamed, a retry would repeat delivered output, so a later
2186        // overflow event keeps today's semantics and surfaces during aggregation.
2187        let (_server, calls, result) = run_streaming_candidates(
2188            "data: {\"id\":\"turn\",\"model\":\"weak\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"partial\"},\"finish_reason\":null}]}\n\ndata: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens).\",\"object\":\"error\",\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
2189        )
2190        .await;
2191        let (_, response) = result?;
2192        assert_eq!(&*calls.lock(), &["weak"]);
2193        let error = response
2194            .llm_response
2195            .into_agg()
2196            .await
2197            .expect_err("the mid-stream overflow should surface to the consumer");
2198        assert!(
2199            error
2200                .to_string()
2201                .contains("exceeds the maximum allowed length")
2202        );
2203
2204        // A stream that ends before any event is not a classified failure; it still
2205        // aggregates to an empty response.
2206        let (_server, calls, result) = run_streaming_candidates("data: [DONE]\n\n").await;
2207        let (_, response) = result?;
2208        assert_eq!(&*calls.lock(), &["weak"]);
2209        let aggregate = response
2210            .llm_response
2211            .into_agg()
2212            .await
2213            .map_err(|source| LibsyError::client_call("weak", source))?;
2214        assert_eq!(completion_text(&aggregate), "");
2215        Ok(())
2216    }
2217}