1use std::collections::HashMap;
21use std::sync::Arc;
22use std::time::Instant;
23
24use futures::{StreamExt, TryStreamExt, stream};
25use http::StatusCode;
26use parking_lot::Mutex;
27use serde_json::{Value, json};
28use switchyard_libsy::{
29 Algorithm, CallModel, LibsyError, OutcomeMetadata, Result, RoutingOutcome, RuntimeModels, drive,
30};
31use switchyard_protocol::{
32 AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message,
33 ModelId, Request, Response, ResponseAccumulator, RoutedLlmClient, RoutingFallbackReason,
34 WireFormat,
35};
36use switchyard_translation::prepare_request_for_target;
37
38use crate::observation::{LlmCallObservation, RunObservation, RunObserver};
39use crate::{metrics, observability};
40
41pub async fn run(
55 algorithm: Arc<dyn Algorithm>,
56 clients: ClientRouter,
57 request: Request,
58 models: Arc<RuntimeModels>,
59 observer: Option<RunObserver>,
60) -> Result<(ModelId, Response)> {
61 let algorithm_name = algorithm.name().to_string();
62 let run_started = Instant::now();
63 let routing_clients = clients.clone();
64 let routing_observations = observer.as_ref().map(|_| Arc::new(Mutex::new(Vec::new())));
67 let outcome = match clients.stored_state_owner(&request) {
68 Some(owner) => Ok(continue_on(owner, &algorithm_name, request)),
69 None => {
70 drive(algorithm, request, models, {
71 let routing_observations = routing_observations.clone();
72 move |call| serve(routing_clients.clone(), call, routing_observations.clone())
73 })
74 .await
75 }
76 };
77 let answered_model = outcome
78 .as_ref()
79 .ok()
80 .and_then(|outcome| outcome.response.as_ref())
81 .and_then(Response::served_model);
82 emit_routing_observations(&observer, &routing_observations, answered_model);
83 let outcome = outcome?;
84 let overhead = run_started.elapsed();
85 metrics::record_routing_overhead(&algorithm_name, overhead);
86
87 let selected_model_id = outcome.selected_model_id()?.clone();
88 if let Some(observer) = &observer
89 && let Some(metadata) = outcome.metadata
90 {
91 observer(RunObservation::Outcome(metadata));
92 }
93 let result = if let Some(response) = outcome.response {
94 let result = clients.remember_state_owner(&outcome.request, response);
95 let served_model = result
96 .as_ref()
97 .ok()
98 .and_then(Response::served_model)
99 .unwrap_or(&selected_model_id);
100 let served_model = served_model.clone();
101 metrics::observe_routed_request(&algorithm_name, &served_model, None, result)
102 } else {
103 let observe = |observation| {
104 if let Some(observer) = &observer {
105 observer(RunObservation::AnswerCall(observation));
106 }
107 };
108 call_first_available(
109 &clients,
110 &algorithm_name,
111 &outcome.request,
112 &outcome.selected_model_ids,
113 &observe,
114 )
115 .await
116 .and_then(|response| clients.remember_state_owner(&outcome.request, response))
117 };
118 if let Some(observer) = &observer {
119 observer(RunObservation::RoutingOverhead(overhead));
120 }
121 result.map(|response| (selected_model_id, response))
122}
123
124pub async fn decide(
129 algorithm: Arc<dyn Algorithm>,
130 clients: ClientRouter,
131 request: Request,
132 models: Arc<RuntimeModels>,
133) -> Result<RoutingOutcome> {
134 let routing_clients = clients.clone();
135 let mut outcome = match clients.stored_state_owner(&request) {
136 Some(owner) => continue_on(owner, algorithm.name(), request),
137 None => {
138 drive(algorithm, request, models, move |call| {
139 serve(routing_clients.clone(), call, None)
140 })
141 .await?
142 }
143 };
144 let selected_model_id = outcome.selected_model_id()?.clone();
145 outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id);
146 outcome.response = outcome
147 .response
148 .map(|response| clients.remember_state_owner(&outcome.request, response))
149 .transpose()?;
150 Ok(outcome)
151}
152
153fn emit_routing_observations(
155 observer: &Option<RunObserver>,
156 observations: &Option<Arc<Mutex<Vec<LlmCallObservation>>>>,
157 answered_model: Option<&ModelId>,
158) {
159 let (Some(observer), Some(observations)) = (observer, observations) else {
160 return;
161 };
162 let mut answer_observed = false;
163 for observation in observations.lock().drain(..) {
164 if !answer_observed && answered_model == Some(&observation.selected_model) {
165 answer_observed = true;
166 observer(RunObservation::AnswerCall(observation));
167 } else {
168 observer(RunObservation::LlmCall(observation));
169 }
170 }
171}
172
173async fn serve(
177 clients: ClientRouter,
178 call: CallModel,
179 observations: Option<Arc<Mutex<Vec<LlmCallObservation>>>>,
180) -> Result<()> {
181 let observe = |observation| {
182 if let Some(observations) = &observations {
183 observations.lock().push(observation);
184 }
185 };
186 let target = call.models.first().ok_or(LibsyError::NoTargets)?;
187 let request = clients.prepare_routing_request(call.request.clone(), target);
188 let response = call_one(
189 &clients,
190 target,
191 request,
192 &call.algorithm,
193 &observe,
194 0,
195 call.models.len(),
196 true,
197 )
198 .await?;
199 call.respond(Ok(response))
200}
201
202async fn call_first_available(
204 clients: &ClientRouter,
205 algorithm: &str,
206 request: &Request,
207 models: &[ModelId],
208 observe: &(dyn Fn(LlmCallObservation) + Send + Sync),
209) -> Result<Response> {
210 for (index, target) in models.iter().enumerate() {
211 let request = clients.prepare_completion_request(request.clone(), target);
212 match call_one(
213 clients,
214 target,
215 request,
216 algorithm,
217 observe,
218 index,
219 models.len(),
220 false,
221 )
222 .await
223 {
224 Ok(response) => return Ok(response),
225 Err(error) if index + 1 == models.len() => return Err(error),
226 Err(error) => match fallback_reason(&error) {
227 Some(reason) => tracing::info!(
228 from = %target,
229 to = %models[index + 1],
230 reason = reason.as_str(),
231 "model call failed; trying next candidate"
232 ),
233 None => return Err(error),
234 },
235 }
236 }
237 Err(LibsyError::NoTargets)
238}
239
240#[allow(clippy::too_many_arguments)]
242#[tracing::instrument(
243 target = "libsy",
244 name = "libsy.client_call",
245 skip_all,
246 fields(
247 algorithm = algorithm,
248 switchyard.algorithm = algorithm,
249 switchyard.candidate = index + 1,
250 switchyard.candidate_count = count,
251 selected_model = %model_id,
252 otel.kind = "client",
253 otel.name = %format_args!("chat {model_id}"),
254 openinference.span.kind = "LLM",
255 gen_ai.operation.name = "chat",
256 gen_ai.request.model = %model_id,
257 gen_ai.request.stream = tracing::field::Empty,
258 gen_ai.request.temperature = tracing::field::Empty,
259 gen_ai.request.top_p = tracing::field::Empty,
260 gen_ai.request.top_k = tracing::field::Empty,
261 gen_ai.request.max_tokens = tracing::field::Empty,
262 gen_ai.request.reasoning.level = tracing::field::Empty,
263 gen_ai.output.type = tracing::field::Empty,
264 gen_ai.conversation.id = tracing::field::Empty,
265 server.address = tracing::field::Empty,
266 server.port = tracing::field::Empty,
267 gen_ai.response.id = tracing::field::Empty,
268 gen_ai.response.model = tracing::field::Empty,
269 gen_ai.usage.input_tokens = tracing::field::Empty,
270 gen_ai.usage.output_tokens = tracing::field::Empty,
271 gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
272 gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
273 gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
274 outcome = tracing::field::Empty,
275 otel.status_code = tracing::field::Empty,
276 error.type = tracing::field::Empty,
277 error = tracing::field::Empty,
278 )
279)]
280async fn call_one(
281 clients: &ClientRouter,
282 model_id: &ModelId,
283 request: Request,
284 algorithm: &str,
285 observe: &(dyn Fn(LlmCallObservation) + Send + Sync),
286 index: usize,
288 count: usize,
290 buffer: bool,
291) -> Result<Response> {
292 let span = tracing::Span::current();
293 observability::record_gen_ai_request(&span, &request.llm_request);
294 if let Some(session_id) = request
295 .metadata
296 .as_ref()
297 .and_then(|metadata| metadata.session_id.as_deref())
298 {
299 span.record("gen_ai.conversation.id", session_id);
300 }
301 let client = clients.route(model_id);
304 let started = Instant::now();
305 let result = async {
306 let mut response = client?.call(request).await?;
307 if buffer && let LlmResponse::Stream(chunks) = response.llm_response {
308 response.llm_response = LlmResponse::Stream(buffer_routing_stream(chunks).await?);
309 }
310 Ok(response)
311 }
312 .await
313 .map_err(|source| LibsyError::client_call(model_id.clone(), source));
314 let duration = started.elapsed();
315
316 let result = result.map(|mut response| {
317 response.set_served_model(model_id);
318 response
319 });
320 let result = observability::observe_client_call(result);
321 let result = if !buffer {
322 metrics::observe_routed_request(algorithm, model_id, Some(duration), result)
323 } else {
324 result
325 };
326 observe(LlmCallObservation {
327 selected_model: model_id.clone(),
328 is_success: result.is_ok(),
329 duration,
330 usage: result
331 .as_ref()
332 .ok()
333 .and_then(|response| response.llm_response.as_agg())
334 .map(|response| response.usage.clone()),
335 });
336 result
337}
338
339async fn buffer_routing_stream(
341 mut chunks: LlmResponseStream,
342) -> std::result::Result<LlmResponseStream, LlmClientError> {
343 let mut events = Vec::new();
344 while let Some(event) = chunks.try_next().await? {
345 for chunk in event.normalized() {
346 match chunk {
347 LlmResponseChunk::DecodeError { message } => {
348 return Err(LlmClientError::ResponseTranslation(message.clone()));
349 }
350 LlmResponseChunk::StreamError { message } => {
351 return Err(LlmClientError::UpstreamHttp {
352 status: StatusCode::BAD_GATEWAY,
353 body: message.clone(),
354 });
355 }
356 _ => {}
357 }
358 }
359 events.push(event);
360 }
361 Ok(stream::iter(events.into_iter().map(Ok)).boxed())
362}
363
364fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
366 let LibsyError::ClientCall { source, .. } = error else {
367 return None;
368 };
369 match source {
370 LlmClientError::ContextWindowExceeded { .. } => Some(RoutingFallbackReason::ContextWindow),
371 LlmClientError::Transport { .. } => Some(RoutingFallbackReason::Unavailable),
372 LlmClientError::UpstreamHttp { status, body }
375 if *status == StatusCode::BAD_REQUEST
376 && serde_json::from_str::<serde_json::Value>(body).is_ok_and(|value| {
377 value["error"]["code"].as_str() == Some("content_policy_violation")
378 }) =>
379 {
380 Some(RoutingFallbackReason::Unavailable)
381 }
382 LlmClientError::UpstreamHttp { status, .. }
383 if matches!(
384 *status,
385 StatusCode::FORBIDDEN | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS
386 ) || status.is_server_error() =>
387 {
388 Some(RoutingFallbackReason::Unavailable)
389 }
390 _ => None,
391 }
392}
393
394fn continue_on(owner: StateOwner, algorithm: &str, mut request: Request) -> RoutingOutcome {
396 tracing::debug!(
397 target = %owner.model,
398 trigger = owner.trigger,
399 "routing Responses continuation to the model that holds its state"
400 );
401 if let Some(history) = owner.history {
402 let mut messages = Vec::with_capacity(history.len + request.llm_request.messages.len());
403 history.extend(&mut messages);
404 messages.append(&mut request.llm_request.messages);
405 request.llm_request.messages = messages;
406 }
407 let mut outcome = RoutingOutcome::route_to(owner.model, Vec::new(), request);
408 outcome.metadata = Some(OutcomeMetadata::new(
409 algorithm.to_string(),
410 Some(json!({"source": "stored_state", "trigger": owner.trigger})),
411 ));
412 outcome
413}
414
415struct StateOwner {
417 model: ModelId,
418 trigger: &'static str,
419 history: Option<Arc<MessageHistory>>,
420}
421
422struct MessageHistory {
423 parent: Option<Arc<Self>>,
424 segment: Arc<[Message]>,
425 len: usize,
426}
427
428impl MessageHistory {
429 fn new(parent: Option<Arc<Self>>, segment: Arc<[Message]>) -> Self {
430 let len = parent.as_ref().map_or(0, |history| history.len) + segment.len();
431 Self {
432 parent,
433 segment,
434 len,
435 }
436 }
437
438 fn extend(&self, messages: &mut Vec<Message>) {
439 let mut segments = Vec::new();
440 let mut history = Some(self);
441 while let Some(node) = history {
442 segments.push(node.segment.as_ref());
443 history = node.parent.as_deref();
444 }
445 for segment in segments.into_iter().rev() {
446 messages.extend_from_slice(segment);
447 }
448 }
449}
450
451#[derive(Clone)]
452struct StoredState {
453 model: ModelId,
454 history: Option<Arc<MessageHistory>>,
456}
457
458#[derive(Clone)]
459struct CanonicalInput {
460 parent: Option<Arc<MessageHistory>>,
461 messages: Arc<[Message]>,
462}
463
464#[derive(Default)]
469struct StateOwners {
470 by_id: HashMap<String, StoredState>,
471}
472
473const MAX_STATE_OWNERS: usize = 65_536;
474
475impl StateOwners {
476 fn owner(&self, id: &str) -> Option<&StoredState> {
477 self.by_id.get(id)
478 }
479
480 fn remember(
481 &mut self,
482 response_id: Option<&str>,
483 conversation_id: Option<&str>,
484 model: &ModelId,
485 history: Option<Arc<MessageHistory>>,
486 ) -> std::result::Result<(), LlmClientError> {
487 let materialized = history.is_some();
488 let ids = [
489 response_id,
490 conversation_id.filter(|id| Some(*id) != response_id),
491 ];
492 let mut new_ids = 0;
493 for id in ids.into_iter().flatten() {
494 match self.by_id.get(id) {
495 Some(owner) if owner.model != *model && !materialized => {
496 return Err(LlmClientError::ResponseStateConflict);
497 }
498 None => new_ids += 1,
499 _ => {}
500 }
501 }
502 if self.by_id.len() + new_ids > MAX_STATE_OWNERS {
503 return Err(LlmClientError::ResponseStateLimitExceeded {
504 limit: MAX_STATE_OWNERS,
505 });
506 }
507 let state = StoredState {
508 model: model.clone(),
509 history,
510 };
511 if let Some(id) = response_id {
512 if materialized {
513 self.by_id.insert(id.to_owned(), state.clone());
514 } else {
515 self.by_id
516 .entry(id.to_owned())
517 .or_insert_with(|| state.clone());
518 }
519 }
520 if let Some(id) = conversation_id {
521 self.by_id.entry(id.to_owned()).or_insert(state);
522 }
523 Ok(())
524 }
525}
526
527fn conversation_id(fields: &serde_json::Map<String, Value>) -> Option<&str> {
529 let conversation = fields.get("conversation")?;
530 conversation
531 .as_str()
532 .or_else(|| conversation.get("id").and_then(Value::as_str))
533}
534
535#[derive(Clone)]
544pub struct ClientRouter {
545 inner: Arc<ClientRouting>,
546}
547
548struct ClientRouting {
549 routing: Routing,
550 target_prompts: HashMap<ModelId, String>,
551 routing_answer_target: Option<ModelId>,
552 state_owners: Mutex<StateOwners>,
554 track_provider_state: bool,
556}
557
558enum Routing {
559 Single(Arc<dyn RoutedLlmClient>),
561 ByModel(HashMap<ModelId, Arc<dyn RoutedLlmClient>>),
563}
564
565impl ClientRouter {
566 pub fn new(by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>) -> Self {
568 Self::new_with_target_prompts(by_model, HashMap::new(), None)
569 }
570
571 pub fn new_with_target_prompts(
580 by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
581 target_prompts: HashMap<ModelId, String>,
582 routing_answer_target: Option<ModelId>,
583 ) -> Self {
584 let completion_targets = by_model.keys().cloned().collect::<Vec<_>>();
585 Self::new_with_completion_targets(
586 by_model,
587 target_prompts,
588 routing_answer_target,
589 &completion_targets,
590 )
591 }
592
593 pub fn new_with_completion_targets(
601 by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
602 target_prompts: HashMap<ModelId, String>,
603 routing_answer_target: Option<ModelId>,
604 completion_targets: &[ModelId],
605 ) -> Self {
606 let mut clients = completion_targets
607 .iter()
608 .filter_map(|model| by_model.get(model));
609 let spans_providers = clients
610 .next()
611 .is_some_and(|first| clients.any(|client| !Arc::ptr_eq(first, client)));
612 Self {
613 inner: Arc::new(ClientRouting {
614 routing: Routing::ByModel(by_model),
615 target_prompts,
616 routing_answer_target,
617 state_owners: Mutex::default(),
618 track_provider_state: spans_providers,
619 }),
620 }
621 }
622
623 pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {
629 Self {
630 inner: Arc::new(ClientRouting {
631 routing: Routing::Single(client),
632 target_prompts: HashMap::new(),
633 routing_answer_target: None,
634 state_owners: Mutex::default(),
635 track_provider_state: false,
636 }),
637 }
638 }
639
640 pub fn route(
645 &self,
646 model: &ModelId,
647 ) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {
648 match &self.inner.routing {
649 Routing::Single(client) => Ok(client),
650 Routing::ByModel(by_model) => {
651 by_model
652 .get(model)
653 .ok_or_else(|| LlmClientError::Configuration {
654 message: format!("no llm client is configured for model {model:?}"),
655 })
656 }
657 }
658 }
659
660 fn stored_state_owner(&self, request: &Request) -> Option<StateOwner> {
662 let fields = &request.llm_request.extensions.fields;
663 let (trigger, id) = match fields.get("previous_response_id").and_then(Value::as_str) {
664 Some(id) => ("previous_response_id", id),
665 None => ("conversation", conversation_id(fields)?),
666 };
667 let state = self.inner.state_owners.lock().owner(id)?.clone();
668 Some(StateOwner {
669 model: state.model,
670 trigger,
671 history: state.history,
672 })
673 }
674
675 fn canonical_input(&self, request: &Request) -> CanonicalInput {
676 let parent = request
677 .llm_request
678 .extensions
679 .fields
680 .get("previous_response_id")
681 .and_then(Value::as_str)
682 .and_then(|id| self.inner.state_owners.lock().owner(id)?.history.clone());
683 let parent_len = parent.as_ref().map_or(0, |history| history.len);
684 let (parent, messages) = match request.llm_request.messages.get(parent_len..) {
685 Some(messages) => (parent, messages.to_vec()),
686 None => (None, request.llm_request.messages.clone()),
687 };
688 CanonicalInput {
689 parent,
690 messages: Arc::from(messages),
691 }
692 }
693
694 fn remember_state_owner(&self, request: &Request, mut response: Response) -> Result<Response> {
696 let Some(model) = response.served_model().cloned() else {
697 return Ok(response);
698 };
699 let fields = &request.llm_request.extensions.fields;
700 let store = fields.get("store").and_then(Value::as_bool) != Some(false);
701 let conversation = conversation_id(fields).map(str::to_owned);
702 let responses_format = WireFormat::OpenAiResponses.into();
703 let responses_request = request
704 .llm_request
705 .preservation
706 .requests
707 .contains_key(&responses_format)
708 || fields.contains_key("previous_response_id")
709 || fields.contains_key("conversation");
710 if !responses_request && !self.inner.track_provider_state {
711 return Ok(response);
712 }
713 let canonical_input = responses_request.then(|| self.canonical_input(request));
714 response.llm_response = match response.llm_response {
715 LlmResponse::Agg(agg) => {
716 if let Some(body) = agg.preservation.responses.get(&responses_format) {
717 if self.inner.track_provider_state {
718 self.remember_response(body, &model, store, conversation.as_deref())
719 .map_err(|error| LibsyError::client_call(model.clone(), error))?;
720 }
721 } else if let Some(input) = &canonical_input {
722 self.remember_canonical_response(&agg, &model, store, input)
723 .map_err(|error| LibsyError::client_call(model.clone(), error))?;
724 }
725 LlmResponse::Agg(agg)
726 }
727 LlmResponse::Stream(stream) => {
728 let router = self.clone();
729 LlmResponse::Stream(Box::pin(futures_util::stream::try_unfold(
730 (
731 stream,
732 router,
733 model,
734 conversation,
735 canonical_input,
736 ResponseAccumulator::new(),
737 false,
738 true,
739 ),
740 move |(
741 mut stream,
742 router,
743 model,
744 conversation,
745 canonical_input,
746 mut accumulator,
747 mut native_responses,
748 mut valid,
749 )| async move {
750 let Some(event) = stream.next().await else {
751 if let Some(input) = &canonical_input
752 && !native_responses
753 && valid
754 {
755 router.remember_canonical_response(
756 &accumulator.finish(),
757 &model,
758 store,
759 input,
760 )?;
761 }
762 return Ok(None);
763 };
764 let event = event?;
765 if let Some(preserved) = event.preservation()
766 && preserved.source().as_str() == WireFormat::OpenAiResponses.as_str()
767 {
768 native_responses = true;
769 if router.inner.track_provider_state
770 && let Some(body) = preserved.raw().get("response")
771 {
772 router.remember_response(
773 body,
774 &model,
775 store,
776 conversation.as_deref(),
777 )?;
778 }
779 }
780 if responses_request {
781 for chunk in event.normalized() {
782 if matches!(
783 chunk,
784 LlmResponseChunk::DecodeError { .. }
785 | LlmResponseChunk::StreamError { .. }
786 ) {
787 valid = false;
788 } else {
789 accumulator.push(chunk.clone());
790 }
791 }
792 }
793 Ok(Some((
794 event,
795 (
796 stream,
797 router,
798 model,
799 conversation,
800 canonical_input,
801 accumulator,
802 native_responses,
803 valid,
804 ),
805 )))
806 },
807 )))
808 }
809 };
810 Ok(response)
811 }
812
813 fn remember_response(
816 &self,
817 body: &Value,
818 model: &ModelId,
819 store: bool,
820 conversation: Option<&str>,
821 ) -> std::result::Result<(), LlmClientError> {
822 let mut owners = self.inner.state_owners.lock();
823 let response_id = body
824 .get("id")
825 .and_then(Value::as_str)
826 .filter(|_| body.get("store").and_then(Value::as_bool).unwrap_or(store));
827 let conversation = body.as_object().and_then(conversation_id).or(conversation);
828 if let Err(error) = owners.remember(response_id, conversation, model, None) {
829 if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
830 tracing::warn!(%error, "Responses state owner capacity reached; state was not retained");
831 } else {
832 tracing::error!(%error, "could not record Responses state owner");
833 return Err(error);
834 }
835 }
836 Ok(())
837 }
838
839 fn remember_canonical_response(
840 &self,
841 response: &AggLlmResponse,
842 model: &ModelId,
843 store: bool,
844 input: &CanonicalInput,
845 ) -> std::result::Result<(), LlmClientError> {
846 let response_id = response.id.as_deref().filter(|_| store);
847 if response_id.is_none() {
848 return Ok(());
849 }
850 let mut segment = input.messages.to_vec();
851 segment.extend(response.outputs.iter().map(|output| Message {
852 role: output.role,
853 content: output.content.clone(),
854 }));
855 let history = Arc::new(MessageHistory::new(
856 input.parent.clone(),
857 Arc::from(segment),
858 ));
859 let result =
860 self.inner
861 .state_owners
862 .lock()
863 .remember(response_id, None, model, Some(history));
864 if let Err(error) = result {
865 if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
866 tracing::warn!(%error, "cross-format Responses state capacity reached; history was not retained");
867 return Ok(());
868 }
869 tracing::error!(%error, "could not record cross-format Responses state");
870 return Err(error);
871 }
872 Ok(())
873 }
874
875 fn prepare_completion_request(&self, mut request: Request, target: &ModelId) -> Request {
877 let prompt = self.inner.target_prompts.get(target).map(String::as_str);
878 prepare_request_for_target(&mut request.llm_request, target, prompt);
879 request
880 }
881
882 fn prepare_routing_request(&self, mut request: Request, target: &ModelId) -> Request {
884 let prompt = if self.inner.routing_answer_target.as_ref() == Some(target) {
885 self.inner.target_prompts.get(target).map(String::as_str)
886 } else {
887 None
888 };
889 prepare_request_for_target(&mut request.llm_request, target, prompt);
890 request
891 }
892}
893
894impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {
895 fn from_iter<I: IntoIterator<Item = (ModelId, Arc<dyn RoutedLlmClient>)>>(iter: I) -> Self {
896 Self::new(iter.into_iter().collect())
897 }
898}
899
900#[cfg(test)]
901mod tests {
902 use std::collections::BTreeMap;
903
904 use super::*;
905 use async_trait::async_trait;
906 use futures::StreamExt;
907 use http::StatusCode;
908 use switchyard_libsy::{Driver, RoutingOutcome};
909 use switchyard_protocol::{
910 ContentBlock, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Message, Role,
911 ToolResult, completion_text, text_request, text_response,
912 };
913 use wiremock::matchers::method;
914 use wiremock::{Mock, MockServer, ResponseTemplate};
915
916 use crate::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
917 use switchyard_protocol::Category;
918
919 struct CandidateAlgorithm {}
920
921 struct AnsweredAlgorithm {
922 model: ModelId,
923 }
924
925 #[async_trait]
926 impl Algorithm for CandidateAlgorithm {
927 fn name(&self) -> &str {
928 "candidate_test"
929 }
930
931 async fn route(
932 self: Arc<Self>,
933 driver: Driver,
934 request: Request,
935 ) -> Result<RoutingOutcome> {
936 let models = driver.models_for(&Category::Any);
937 let selected_model = models.first().cloned().ok_or(LibsyError::NoTargets)?;
938 Ok(RoutingOutcome::route_to(
939 selected_model,
940 models.iter().skip(1).cloned().collect(),
941 request,
942 ))
943 }
944 }
945
946 #[async_trait]
947 impl Algorithm for AnsweredAlgorithm {
948 fn name(&self) -> &str {
949 "answered_test"
950 }
951
952 async fn route(
953 self: Arc<Self>,
954 driver: Driver,
955 request: Request,
956 ) -> Result<RoutingOutcome> {
957 let response = driver
958 .call_model(request.clone(), vec![self.model.clone()])
959 .await?;
960 Ok(RoutingOutcome::answered(
961 self.model.clone(),
962 request,
963 response,
964 ))
965 }
966 }
967
968 #[derive(Clone, Copy)]
969 enum FirstOutcome {
970 ContextWindow,
971 ContentPolicy,
972 Unauthorized,
973 StreamSuccess,
974 MidStreamError,
975 }
976
977 struct CandidateClient {
978 calls: Mutex<Vec<ModelId>>,
979 requests: Mutex<Vec<Request>>,
980 first: FirstOutcome,
981 }
982
983 #[async_trait]
984 impl RoutedLlmClient for CandidateClient {
985 async fn call(&self, request: Request) -> std::result::Result<Response, LlmClientError> {
986 let model = request.model_id().unwrap_or_default();
987 self.calls.lock().push(model.clone());
988 self.requests.lock().push(request);
989 if model == "weak" {
990 return match self.first {
991 FirstOutcome::ContextWindow => Err(LlmClientError::ContextWindowExceeded {
992 model,
993 message: "too long".to_string(),
994 }),
995 FirstOutcome::ContentPolicy => Err(LlmClientError::UpstreamHttp {
996 status: StatusCode::BAD_REQUEST,
997 body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(),
998 }),
999 FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {
1000 status: StatusCode::UNAUTHORIZED,
1001 body: "unauthorized".to_string(),
1002 }),
1003 FirstOutcome::StreamSuccess => Ok(stream_response(vec![
1004 LlmResponseChunk::TextDelta {
1005 index: 0,
1006 text: "streamed".to_string(),
1007 },
1008 LlmResponseChunk::MessageStop {
1009 reason: Some("stop".to_string()),
1010 },
1011 ])),
1012 FirstOutcome::MidStreamError => Ok(stream_response(vec![
1013 LlmResponseChunk::TextDelta {
1014 index: 0,
1015 text: "partial".to_string(),
1016 },
1017 LlmResponseChunk::StreamError {
1018 message: "stream failed".to_string(),
1019 },
1020 ])),
1021 };
1022 }
1023 Ok(Response {
1024 llm_response: LlmResponse::Agg(text_response(Some(model.to_string()), model)),
1025 metadata: None,
1026 upstream_headers: http::HeaderMap::new(),
1027 })
1028 }
1029 }
1030
1031 fn stream_response(chunks: Vec<LlmResponseChunk>) -> Response {
1032 Response {
1033 llm_response: LlmResponse::Stream(
1034 futures::stream::iter(
1035 chunks
1036 .into_iter()
1037 .map(|chunk| Ok(LlmResponseStreamEvent::from(chunk))),
1038 )
1039 .boxed(),
1040 ),
1041 metadata: None,
1042 upstream_headers: http::HeaderMap::new(),
1043 }
1044 }
1045
1046 fn request() -> Request {
1047 Request {
1048 llm_request: text_request(Some("auto".to_string()), "hello".to_string()),
1049 raw_request: None,
1050 metadata: None,
1051 }
1052 }
1053
1054 fn instruction_text(request: &Request) -> Vec<&str> {
1055 request
1056 .llm_request
1057 .instructions
1058 .iter()
1059 .flat_map(|instruction| &instruction.content)
1060 .filter_map(|block| match block {
1061 ContentBlock::Text { text } => Some(text.as_str()),
1062 _ => None,
1063 })
1064 .collect()
1065 }
1066
1067 async fn run_candidates(
1068 first: FirstOutcome,
1069 ) -> (Arc<CandidateClient>, Result<(ModelId, Response)>) {
1070 let client = Arc::new(CandidateClient {
1071 calls: Mutex::new(Vec::new()),
1072 requests: Mutex::new(Vec::new()),
1073 first,
1074 });
1075 let algorithm = Arc::new(CandidateAlgorithm {});
1076 let models = to_category_map(&["weak", "strong"]);
1077 let result = run(
1078 algorithm,
1079 ClientRouter::single(client.clone()),
1080 request(),
1081 models,
1082 None,
1083 )
1084 .await;
1085 (client, result)
1086 }
1087
1088 fn to_category_map(names: &[&str]) -> Arc<RuntimeModels> {
1089 Arc::new(RuntimeModels::new(
1090 [(
1091 Category::Any,
1092 names.iter().map(|name| ModelId::from(*name)).collect(),
1093 )]
1094 .into(),
1095 ))
1096 }
1097
1098 #[tokio::test]
1099 async fn answered_outcome_does_not_make_a_second_model_call() -> Result<()> {
1100 let client = Arc::new(CandidateClient {
1101 calls: Mutex::new(Vec::new()),
1102 requests: Mutex::new(Vec::new()),
1103 first: FirstOutcome::StreamSuccess,
1104 });
1105 let observations = Arc::new(Mutex::new(Vec::new()));
1106 let observed = Arc::clone(&observations);
1107 let observer: RunObserver = Arc::new(move |event| observed.lock().push(event));
1108
1109 let (selected, response) = run(
1110 Arc::new(AnsweredAlgorithm {
1111 model: "weak".into(),
1112 }),
1113 ClientRouter::single(client.clone()),
1114 request(),
1115 Arc::new(RuntimeModels::default()),
1116 Some(observer),
1117 )
1118 .await?;
1119
1120 assert_eq!(selected, "weak");
1121 assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));
1122 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
1123 let observations = observations.lock();
1124 assert!(matches!(observations[0], RunObservation::AnswerCall(_)));
1125 assert!(matches!(
1126 observations[1],
1127 RunObservation::Outcome(ref metadata) if metadata.algorithm == "answered_test"
1128 ));
1129 assert!(matches!(
1130 observations[2],
1131 RunObservation::RoutingOverhead(_)
1132 ));
1133 assert_eq!(observations.len(), 3);
1134 Ok(())
1135 }
1136
1137 #[test]
1138 fn answer_observation_keeps_call_order() {
1139 let pending = Some(Arc::new(Mutex::new(
1140 ["answer", "judge"]
1141 .map(|model| LlmCallObservation {
1142 selected_model: model.into(),
1143 is_success: true,
1144 duration: std::time::Duration::ZERO,
1145 usage: None,
1146 })
1147 .into(),
1148 )));
1149 let emitted = Arc::new(Mutex::new(Vec::new()));
1150 let captured = Arc::clone(&emitted);
1151 let observer: RunObserver = Arc::new(move |event| captured.lock().push(event));
1152 let answer = ModelId::from("answer");
1153
1154 emit_routing_observations(&Some(observer), &pending, Some(&answer));
1155
1156 assert!(matches!(
1157 &emitted.lock()[..],
1158 [RunObservation::AnswerCall(answer), RunObservation::LlmCall(judge)]
1159 if answer.selected_model == "answer" && judge.selected_model == "judge"
1160 ));
1161 }
1162
1163 #[tokio::test]
1164 async fn responses_state_tracking_uses_upstream_storage_and_format() -> Result<()> {
1165 for (responses, store, conversation) in [
1166 (true, true, None),
1167 (true, false, None),
1168 (true, false, Some(json!("conv_plain"))),
1169 (true, false, Some(json!({"id": "conv_object"}))),
1170 (false, true, None),
1171 ] {
1172 for stream in [false, true] {
1173 let server = MockServer::start().await;
1174 Mock::given(method("POST"))
1175 .respond_with(move |request: &wiremock::Request| {
1176 let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
1177 let response = if responses {
1178 json!({
1179 "id": "resp_seed", "object": "response", "model": "weak",
1180 "status": "completed", "output": [], "store": body["store"],
1181 "conversation": body["conversation"],
1182 })
1183 } else if stream {
1184 json!({"id": "resp_seed", "model": "weak", "choices": [{"index": 0, "delta": {"content": "ok"}, "finish_reason": "stop"}]})
1185 } else {
1186 json!({"id": "resp_seed", "model": "weak", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}]})
1187 };
1188 if stream {
1189 let event = if responses { json!({"type": "response.completed", "response": response}) } else { response };
1190 ResponseTemplate::new(200)
1191 .insert_header("content-type", "text/event-stream")
1192 .set_body_string(if responses {
1193 format!("event: response.completed\ndata: {event}\n\n")
1194 } else {
1195 format!("data: {event}\n\ndata: [DONE]\n\n")
1196 })
1197 } else {
1198 ResponseTemplate::new(200).set_body_json(response)
1199 }
1200 })
1201 .mount(&server).await;
1202 let mut clients = HashMap::new();
1203 for model in ["weak", "strong"] {
1204 let config = HttpBackendConfig {
1205 base_url: server.uri(),
1206 api_key: None,
1207 forward_auth: false,
1208 extra_headers: BTreeMap::new(),
1209 extra_body: BTreeMap::from([("store".to_string(), json!(store))]),
1210 reasoning_effort: None,
1211 max_retries: 0,
1212 timeout: None,
1213 };
1214 let backend = if responses {
1215 Backend::OpenAiResponses(config)
1216 } else {
1217 Backend::OpenAiChat(config)
1218 };
1219 let client: Arc<dyn RoutedLlmClient> = Arc::new(
1220 TranslatingLlmClient::new(&[ModelConfig::new(model, backend, None)])
1221 .map_err(|error| LibsyError::external("building test client", error))?,
1222 );
1223 clients.insert(ModelId::from(model), client);
1224 }
1225 let clients = ClientRouter::new(clients);
1226 let mut seed = request();
1227 seed.llm_request.stream = stream;
1228 if let Some(conversation) = &conversation {
1229 seed.llm_request
1230 .extensions
1231 .fields
1232 .insert("conversation".to_string(), conversation.clone());
1233 }
1234 let (_, response) = run(
1235 Arc::new(switchyard_libsy::Passthrough),
1236 clients.clone(),
1237 seed,
1238 to_category_map(&["weak"]),
1239 None,
1240 )
1241 .await?;
1242 response
1243 .llm_response
1244 .into_agg()
1245 .await
1246 .map_err(|error| LibsyError::client_call("weak", error))?;
1247 let mut continuations = vec![(
1248 "previous_response_id",
1249 json!("resp_seed"),
1250 responses && store,
1251 )];
1252 if let Some(conversation) = conversation.clone() {
1253 continuations.push(("conversation", conversation, true));
1254 }
1255 for (field, id, pinned) in continuations {
1256 let mut follow = request();
1257 follow
1258 .llm_request
1259 .extensions
1260 .fields
1261 .insert(field.to_string(), id);
1262 let outcome = decide(
1263 Arc::new(switchyard_libsy::Passthrough),
1264 clients.clone(),
1265 follow,
1266 to_category_map(&["strong"]),
1267 )
1268 .await?;
1269 assert_eq!(
1270 outcome.selected_model_id()?.as_str(),
1271 if pinned { "weak" } else { "strong" },
1272 "responses={responses}, store={store}, stream={stream}, field={field}"
1273 );
1274 }
1275 let requests = server.received_requests().await.expect("request recording");
1276 assert_eq!(requests.len(), 1);
1277 assert_eq!(
1278 serde_json::from_slice::<Value>(&requests[0].body).expect("request JSON")["store"],
1279 store
1280 );
1281 }
1282 }
1283 Ok(())
1284 }
1285
1286 #[tokio::test]
1287 async fn responses_stored_tool_continuation_materializes_for_anthropic() -> Result<()> {
1288 let server = MockServer::start().await;
1289 Mock::given(method("POST"))
1290 .respond_with(|request: &wiremock::Request| {
1291 let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
1292 let blocks = body["messages"]
1293 .as_array()
1294 .into_iter()
1295 .flatten()
1296 .filter_map(|message| message["content"].as_array())
1297 .flatten()
1298 .collect::<Vec<_>>();
1299 let has_result = blocks.iter().any(|block| block["type"] == "tool_result");
1300 let has_call = blocks.iter().any(|block| block["type"] == "tool_use");
1301 let has_thanks = blocks
1302 .iter()
1303 .any(|block| block["type"] == "text" && block["text"] == "thanks")
1304 || body["messages"].as_array().is_some_and(|messages| {
1305 messages
1306 .iter()
1307 .any(|message| message["content"] == "thanks")
1308 });
1309 if has_result && !has_call {
1310 return ResponseTemplate::new(400).set_body_json(json!({
1311 "type": "error",
1312 "error": {
1313 "type": "invalid_request_error",
1314 "message": "tool_result has no corresponding tool_use"
1315 }
1316 }));
1317 }
1318 let response = if has_thanks {
1319 json!({
1320 "id": "msg_third", "type": "message", "role": "assistant",
1321 "model": "weak", "content": [{"type": "text", "text": "done"}],
1322 "stop_reason": "end_turn", "usage": {"input_tokens": 6, "output_tokens": 1}
1323 })
1324 } else if has_result {
1325 json!({
1326 "id": "msg_follow", "type": "message", "role": "assistant",
1327 "model": "weak", "content": [{"type": "text", "text": "sunny"}],
1328 "stop_reason": "end_turn", "usage": {"input_tokens": 4, "output_tokens": 1}
1329 })
1330 } else {
1331 json!({
1332 "id": "msg_seed", "type": "message", "role": "assistant",
1333 "model": "weak",
1334 "content": [{
1335 "type": "tool_use", "id": "toolu_weather", "name": "get_weather",
1336 "input": {"city": "Paris"}
1337 }],
1338 "stop_reason": "tool_use", "usage": {"input_tokens": 3, "output_tokens": 1}
1339 })
1340 };
1341 ResponseTemplate::new(200).set_body_json(response)
1342 })
1343 .mount(&server)
1344 .await;
1345
1346 let client: Arc<dyn RoutedLlmClient> = Arc::new(
1347 TranslatingLlmClient::new(&[ModelConfig::new(
1348 "weak",
1349 Backend::Anthropic(HttpBackendConfig {
1350 base_url: server.uri(),
1351 api_key: None,
1352 forward_auth: false,
1353 extra_headers: BTreeMap::new(),
1354 extra_body: BTreeMap::new(),
1355 reasoning_effort: None,
1356 max_retries: 0,
1357 timeout: None,
1358 }),
1359 None,
1360 )])
1361 .map_err(|error| LibsyError::external("building test client", error))?,
1362 );
1363 let clients = ClientRouter::new(HashMap::from([(ModelId::from("weak"), client)]));
1364 let models = to_category_map(&["weak"]);
1365
1366 let seed = Request {
1367 llm_request: switchyard_translation::decode_request(
1368 WireFormat::OpenAiResponses,
1369 &json!({
1370 "model": "route",
1371 "input": "Call get_weather for Paris",
1372 "tools": [{
1373 "type": "function", "name": "get_weather",
1374 "parameters": {"type": "object"}
1375 }],
1376 "store": true
1377 }),
1378 )
1379 .map_err(|error| LibsyError::external("decoding seed request", error))?,
1380 raw_request: None,
1381 metadata: None,
1382 };
1383 let (_, response) = run(
1384 Arc::new(switchyard_libsy::Passthrough),
1385 clients.clone(),
1386 seed,
1387 models.clone(),
1388 None,
1389 )
1390 .await?;
1391 assert_eq!(
1392 response
1393 .llm_response
1394 .as_agg()
1395 .and_then(|agg| agg.id.as_deref()),
1396 Some("msg_seed")
1397 );
1398
1399 let follow = Request {
1400 llm_request: switchyard_translation::decode_request(
1401 WireFormat::OpenAiResponses,
1402 &json!({
1403 "model": "route",
1404 "previous_response_id": "msg_seed",
1405 "input": [{
1406 "type": "function_call_output",
1407 "call_id": "toolu_weather",
1408 "output": "Paris is sunny"
1409 }],
1410 "tools": [{
1411 "type": "function", "name": "get_weather",
1412 "parameters": {"type": "object"}
1413 }],
1414 "store": true
1415 }),
1416 )
1417 .map_err(|error| LibsyError::external("decoding continuation request", error))?,
1418 raw_request: None,
1419 metadata: None,
1420 };
1421 let (_, response) = run(
1422 Arc::new(switchyard_libsy::Passthrough),
1423 clients.clone(),
1424 follow,
1425 models.clone(),
1426 None,
1427 )
1428 .await?;
1429 assert_eq!(
1430 completion_text(
1431 response
1432 .llm_response
1433 .as_agg()
1434 .expect("buffered Anthropic response"),
1435 ),
1436 "sunny"
1437 );
1438
1439 let third = Request {
1440 llm_request: switchyard_translation::decode_request(
1441 WireFormat::OpenAiResponses,
1442 &json!({
1443 "model": "route",
1444 "previous_response_id": "msg_follow",
1445 "input": "thanks",
1446 "store": true
1447 }),
1448 )
1449 .map_err(|error| LibsyError::external("decoding chained request", error))?,
1450 raw_request: None,
1451 metadata: None,
1452 };
1453 let history = clients
1454 .stored_state_owner(&third)
1455 .and_then(|owner| owner.history)
1456 .expect("chained canonical history");
1457 assert_eq!(history.len, 4);
1458 assert_eq!(history.segment.len(), 2);
1459 assert!(history.parent.is_some());
1460 let (_, response) = run(
1461 Arc::new(switchyard_libsy::Passthrough),
1462 clients,
1463 third,
1464 models,
1465 None,
1466 )
1467 .await?;
1468 assert_eq!(
1469 completion_text(
1470 response
1471 .llm_response
1472 .as_agg()
1473 .expect("buffered Anthropic response"),
1474 ),
1475 "done"
1476 );
1477
1478 let requests = server.received_requests().await.expect("request recording");
1479 assert_eq!(requests.len(), 3);
1480 let follow: Value = serde_json::from_slice(&requests[1].body)
1481 .map_err(|error| LibsyError::external("decoding captured request", error))?;
1482 assert_eq!(follow["messages"][1]["content"][0]["type"], "tool_use");
1483 assert_eq!(follow["messages"][2]["content"][0]["type"], "tool_result");
1484 let third: Value = serde_json::from_slice(&requests[2].body)
1485 .map_err(|error| LibsyError::external("decoding captured request", error))?;
1486 assert_eq!(third["messages"][1]["content"][0]["type"], "tool_use");
1487 assert_eq!(third["messages"][2]["content"][0]["type"], "tool_result");
1488 assert_eq!(third["messages"][3]["content"], "sunny");
1489 assert_eq!(third["messages"][4]["content"][0]["text"], "thanks");
1490 Ok(())
1491 }
1492
1493 #[tokio::test]
1494 async fn streamed_cross_format_state_is_recorded_only_after_completion() -> Result<()> {
1495 let client = Arc::new(CandidateClient {
1496 calls: Mutex::new(Vec::new()),
1497 requests: Mutex::new(Vec::new()),
1498 first: FirstOutcome::StreamSuccess,
1499 });
1500 let clients = ClientRouter::new(HashMap::from([(
1501 ModelId::from("weak"),
1502 client as Arc<dyn RoutedLlmClient>,
1503 )]));
1504 let mut seed = request();
1505 seed.llm_request.messages = vec![Message::text(Role::User, "inspect")];
1506 seed.llm_request
1507 .preservation
1508 .requests
1509 .insert(WireFormat::OpenAiResponses.into(), json!({}));
1510 let mut response = stream_response(vec![
1511 LlmResponseChunk::MessageStart {
1512 id: Some("msg_stream".to_string()),
1513 model: Some("weak".to_string()),
1514 },
1515 LlmResponseChunk::ToolCallDelta {
1516 index: 0,
1517 id: Some("toolu_stream".to_string()),
1518 name: Some("lookup".to_string()),
1519 arguments_delta: Some("{}".to_string()),
1520 },
1521 LlmResponseChunk::MessageStop {
1522 reason: Some("tool_use".to_string()),
1523 },
1524 ]);
1525 response.set_served_model(&ModelId::from("weak"));
1526 let response = clients.remember_state_owner(&seed, response)?;
1527
1528 let mut follow = request();
1529 follow
1530 .llm_request
1531 .extensions
1532 .fields
1533 .insert("previous_response_id".to_string(), json!("msg_stream"));
1534 follow.llm_request.messages = vec![Message {
1535 role: Role::Tool,
1536 content: vec![ContentBlock::ToolResult(ToolResult {
1537 tool_call_id: "toolu_stream".to_string(),
1538 content: vec![ContentBlock::Text {
1539 text: "ready".to_string(),
1540 }],
1541 is_error: None,
1542 })],
1543 }];
1544 assert!(clients.stored_state_owner(&follow).is_none());
1545 response
1546 .llm_response
1547 .into_agg()
1548 .await
1549 .map_err(|error| LibsyError::client_call("weak", error))?;
1550
1551 let outcome = continue_on(
1552 clients
1553 .stored_state_owner(&follow)
1554 .expect("completed stream state"),
1555 "passthrough",
1556 follow,
1557 );
1558 assert_eq!(outcome.request.llm_request.messages.len(), 3);
1559 assert!(matches!(
1560 outcome.request.llm_request.messages[1].content.as_slice(),
1561 [ContentBlock::ToolCall(call)] if call.id == "toolu_stream"
1562 ));
1563 assert!(matches!(
1564 outcome.request.llm_request.messages[2].content.as_slice(),
1565 [ContentBlock::ToolResult(result)] if result.tool_call_id == "toolu_stream"
1566 ));
1567 Ok(())
1568 }
1569
1570 #[test]
1571 fn cross_format_response_with_store_false_is_not_retained() -> Result<()> {
1572 let client = Arc::new(CandidateClient {
1573 calls: Mutex::new(Vec::new()),
1574 requests: Mutex::new(Vec::new()),
1575 first: FirstOutcome::StreamSuccess,
1576 });
1577 let clients = ClientRouter::new(HashMap::from([(
1578 ModelId::from("weak"),
1579 client as Arc<dyn RoutedLlmClient>,
1580 )]));
1581 let mut seed = request();
1582 seed.llm_request
1583 .extensions
1584 .fields
1585 .insert("store".to_string(), json!(false));
1586 seed.llm_request
1587 .preservation
1588 .requests
1589 .insert(WireFormat::OpenAiResponses.into(), json!({}));
1590 let mut agg = text_response(Some("weak".to_string()), "done".to_string());
1591 agg.id = Some("msg_not_stored".to_string());
1592 let mut response = Response {
1593 llm_response: LlmResponse::Agg(agg),
1594 metadata: None,
1595 upstream_headers: http::HeaderMap::new(),
1596 };
1597 response.set_served_model(&ModelId::from("weak"));
1598 clients.remember_state_owner(&seed, response)?;
1599
1600 let mut follow = request();
1601 follow
1602 .llm_request
1603 .extensions
1604 .fields
1605 .insert("previous_response_id".to_string(), json!("msg_not_stored"));
1606 assert!(clients.stored_state_owner(&follow).is_none());
1607 Ok(())
1608 }
1609
1610 #[tokio::test]
1611 async fn state_capacity_does_not_fail_completed_responses() -> Result<()> {
1612 let client = Arc::new(CandidateClient {
1613 calls: Mutex::new(Vec::new()),
1614 requests: Mutex::new(Vec::new()),
1615 first: FirstOutcome::StreamSuccess,
1616 });
1617 let clients = ClientRouter::new(HashMap::from([(
1618 ModelId::from("weak"),
1619 client as Arc<dyn RoutedLlmClient>,
1620 )]));
1621 let model = ModelId::from("weak");
1622 clients.inner.state_owners.lock().by_id = (0..MAX_STATE_OWNERS)
1623 .map(|id| {
1624 (
1625 format!("existing_{id}"),
1626 StoredState {
1627 model: model.clone(),
1628 history: None,
1629 },
1630 )
1631 })
1632 .collect();
1633
1634 let mut request = request();
1635 request
1636 .llm_request
1637 .preservation
1638 .requests
1639 .insert(WireFormat::OpenAiResponses.into(), json!({}));
1640 let mut agg = text_response(Some("weak".to_string()), "done".to_string());
1641 agg.id = Some("overflow_buffered".to_string());
1642 let mut buffered = Response {
1643 llm_response: LlmResponse::Agg(agg),
1644 metadata: None,
1645 upstream_headers: http::HeaderMap::new(),
1646 };
1647 buffered.set_served_model(&model);
1648 clients.remember_state_owner(&request, buffered)?;
1649
1650 let mut streamed = stream_response(vec![
1651 LlmResponseChunk::MessageStart {
1652 id: Some("overflow_streamed".to_string()),
1653 model: Some("weak".to_string()),
1654 },
1655 LlmResponseChunk::TextDelta {
1656 index: 0,
1657 text: "done".to_string(),
1658 },
1659 LlmResponseChunk::MessageStop {
1660 reason: Some("stop".to_string()),
1661 },
1662 ]);
1663 streamed.set_served_model(&model);
1664 clients
1665 .remember_state_owner(&request, streamed)?
1666 .llm_response
1667 .into_agg()
1668 .await
1669 .map_err(|error| LibsyError::client_call("weak", error))?;
1670
1671 assert_eq!(
1672 clients.inner.state_owners.lock().by_id.len(),
1673 MAX_STATE_OWNERS
1674 );
1675 Ok(())
1676 }
1677
1678 #[test]
1679 fn state_owners_preserve_existing_records_at_capacity()
1680 -> std::result::Result<(), LlmClientError> {
1681 let mut owners = StateOwners::default();
1682 let model = ModelId::from("model/a");
1683 for id in 0..MAX_STATE_OWNERS - 1 {
1684 owners.remember(Some(&format!("resp_{id}")), None, &model, None)?;
1685 }
1686 assert!(matches!(
1687 owners.remember(Some("new_response"), Some("new_conversation"), &model, None,),
1688 Err(LlmClientError::ResponseStateLimitExceeded {
1689 limit: MAX_STATE_OWNERS
1690 })
1691 ));
1692 assert!(owners.owner("new_response").is_none());
1693 assert!(owners.owner("new_conversation").is_none());
1694 owners.remember(Some("last_id"), Some("last_id"), &model, None)?;
1695 owners.remember(Some("last_id"), Some("resp_0"), &model, None)?;
1696 assert!(matches!(
1697 owners.remember(Some("over_limit"), None, &model, None),
1698 Err(LlmClientError::ResponseStateLimitExceeded { .. })
1699 ));
1700 assert!(matches!(
1701 owners.remember(Some("resp_0"), None, &ModelId::from("model/b"), None,),
1702 Err(LlmClientError::ResponseStateConflict)
1703 ));
1704 assert_eq!(
1705 owners.owner("resp_0").map(|state| &state.model),
1706 Some(&model)
1707 );
1708 assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
1709 Ok(())
1710 }
1711
1712 #[test]
1713 fn concurrent_state_owners_do_not_exceed_capacity() {
1714 let model = ModelId::from("model/a");
1715 let owners = Arc::new(Mutex::new(StateOwners {
1716 by_id: (0..MAX_STATE_OWNERS - 16)
1717 .map(|id| {
1718 (
1719 format!("existing_{id}"),
1720 StoredState {
1721 model: model.clone(),
1722 history: None,
1723 },
1724 )
1725 })
1726 .collect(),
1727 }));
1728 let barrier = Arc::new(std::sync::Barrier::new(32));
1729 let accepted = std::thread::scope(|scope| {
1730 let handles = (0..32)
1731 .map(|id| {
1732 let owners = owners.clone();
1733 let barrier = barrier.clone();
1734 let model = model.clone();
1735 scope.spawn(move || {
1736 barrier.wait();
1737 let result = owners.lock().remember(
1738 Some(&format!("resp_{id}")),
1739 Some(&format!("conv_{id}")),
1740 &model,
1741 None,
1742 );
1743 match result {
1744 Ok(()) => Some(id),
1745 Err(LlmClientError::ResponseStateLimitExceeded { .. }) => None,
1746 other => panic!("unexpected state tracking result: {other:?}"),
1747 }
1748 })
1749 })
1750 .collect::<Vec<_>>();
1751 handles
1752 .into_iter()
1753 .filter_map(|handle| handle.join().expect("request thread panicked"))
1754 .collect::<Vec<_>>()
1755 });
1756 let owners = owners.lock();
1757 assert_eq!(accepted.len(), 8);
1758 assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
1759 assert_eq!(
1760 owners.owner("existing_0").map(|state| &state.model),
1761 Some(&model)
1762 );
1763 for id in accepted {
1764 assert_eq!(
1765 owners
1766 .owner(&format!("resp_{id}"))
1767 .map(|state| &state.model),
1768 Some(&model)
1769 );
1770 assert_eq!(
1771 owners
1772 .owner(&format!("conv_{id}"))
1773 .map(|state| &state.model),
1774 Some(&model)
1775 );
1776 }
1777 }
1778
1779 #[tokio::test]
1780 async fn each_fallback_candidate_receives_only_its_own_prompt() -> Result<()> {
1781 let client = Arc::new(CandidateClient {
1782 calls: Mutex::new(Vec::new()),
1783 requests: Mutex::new(Vec::new()),
1784 first: FirstOutcome::ContextWindow,
1785 });
1786 let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
1787 let clients = ClientRouter::new_with_target_prompts(
1788 HashMap::from([
1789 (ModelId::from("weak"), Arc::clone(&routed_client)),
1790 (ModelId::from("strong"), routed_client),
1791 ]),
1792 HashMap::from([
1793 ("weak".into(), "weak prompt".to_string()),
1794 ("strong".into(), "strong prompt".to_string()),
1795 ]),
1796 None,
1797 );
1798
1799 run(
1800 Arc::new(CandidateAlgorithm {}),
1801 clients,
1802 request(),
1803 to_category_map(&["weak", "strong"]),
1804 None,
1805 )
1806 .await?;
1807
1808 let calls = client.requests.lock();
1809 assert_eq!(calls.len(), 2);
1810 assert_eq!(instruction_text(&calls[0]), ["weak prompt"]);
1811 assert_eq!(instruction_text(&calls[1]), ["strong prompt"]);
1812 Ok(())
1813 }
1814
1815 #[tokio::test]
1816 async fn decision_prompts_a_routing_response_target() -> Result<()> {
1817 let client = Arc::new(CandidateClient {
1818 calls: Mutex::new(Vec::new()),
1819 requests: Mutex::new(Vec::new()),
1820 first: FirstOutcome::StreamSuccess,
1821 });
1822 let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
1823 let clients = ClientRouter::new_with_target_prompts(
1824 HashMap::from([(ModelId::from("answer"), routed_client)]),
1825 HashMap::from([("answer".into(), "answer prompt".to_string())]),
1826 Some("answer".into()),
1827 );
1828
1829 decide(
1830 Arc::new(AnsweredAlgorithm {
1831 model: "answer".into(),
1832 }),
1833 clients,
1834 request(),
1835 Arc::new(RuntimeModels::default()),
1836 )
1837 .await?;
1838
1839 let calls = client.requests.lock();
1840 assert_eq!(calls.len(), 1);
1841 assert_eq!(instruction_text(&calls[0]), ["answer prompt"]);
1842 Ok(())
1843 }
1844
1845 #[tokio::test]
1846 async fn decision_prepares_only_the_selected_request() -> Result<()> {
1847 let client = Arc::new(CandidateClient {
1848 calls: Mutex::new(Vec::new()),
1849 requests: Mutex::new(Vec::new()),
1850 first: FirstOutcome::StreamSuccess,
1851 });
1852 let routed_client: Arc<dyn RoutedLlmClient> = client;
1853 let clients = ClientRouter::new_with_target_prompts(
1854 HashMap::from([
1855 (ModelId::from("weak"), Arc::clone(&routed_client)),
1856 (ModelId::from("strong"), routed_client),
1857 ]),
1858 HashMap::from([
1859 (ModelId::from("weak"), "weak prompt".to_string()),
1860 (ModelId::from("strong"), "strong prompt".to_string()),
1861 ]),
1862 None,
1863 );
1864
1865 let outcome = decide(
1866 Arc::new(CandidateAlgorithm {}),
1867 clients,
1868 request(),
1869 to_category_map(&["weak", "strong"]),
1870 )
1871 .await?;
1872
1873 assert_eq!(
1874 outcome.selected_model_ids,
1875 [ModelId::from("weak"), ModelId::from("strong")]
1876 );
1877 assert_eq!(instruction_text(&outcome.request), ["weak prompt"]);
1878 Ok(())
1879 }
1880
1881 #[test]
1882 fn fallback_only_accepts_context_policy_and_unavailable_failures() {
1883 let error = |source| LibsyError::client_call("target", source);
1884 assert_eq!(
1885 fallback_reason(&error(LlmClientError::ContextWindowExceeded {
1886 model: "target".into(),
1887 message: "too long".to_string(),
1888 })),
1889 Some(RoutingFallbackReason::ContextWindow)
1890 );
1891 for status in [
1892 StatusCode::FORBIDDEN,
1893 StatusCode::REQUEST_TIMEOUT,
1894 StatusCode::TOO_MANY_REQUESTS,
1895 StatusCode::INTERNAL_SERVER_ERROR,
1896 StatusCode::from_u16(599).expect("599 is a valid status"),
1897 ] {
1898 assert_eq!(
1899 fallback_reason(&error(LlmClientError::UpstreamHttp {
1900 status,
1901 body: "failed".to_string(),
1902 })),
1903 Some(RoutingFallbackReason::Unavailable)
1904 );
1905 }
1906 for (body, expected) in [
1907 (
1908 r#"{"error":{"code":"content_policy_violation"}}"#,
1909 Some(RoutingFallbackReason::Unavailable),
1910 ),
1911 (
1912 r#"{"error":{"code":"invalid_request_error"},"metadata":"content_policy_violation"}"#,
1913 None,
1914 ),
1915 (r#"{"error":{"message":"content_policy_violation"}}"#, None),
1916 ("content_policy_violation", None),
1917 ] {
1918 assert_eq!(
1919 fallback_reason(&error(LlmClientError::UpstreamHttp {
1920 status: StatusCode::BAD_REQUEST,
1921 body: body.to_string(),
1922 })),
1923 expected,
1924 "{body}"
1925 );
1926 }
1927 for status in [
1928 StatusCode::BAD_REQUEST,
1929 StatusCode::UNAUTHORIZED,
1930 StatusCode::NOT_FOUND,
1931 StatusCode::CONFLICT,
1932 StatusCode::from_u16(499).expect("499 is a valid status"),
1933 StatusCode::from_u16(600).expect("600 is a valid status"),
1934 ] {
1935 assert_eq!(
1936 fallback_reason(&error(LlmClientError::UpstreamHttp {
1937 status,
1938 body: "failed".to_string(),
1939 })),
1940 None
1941 );
1942 }
1943 }
1944
1945 #[tokio::test]
1946 async fn candidate_failures_follow_the_fallback_policy() -> Result<()> {
1947 let (client, result) = run_candidates(FirstOutcome::ContextWindow).await;
1949 let (_, response) = result?;
1950 assert_eq!(
1951 &*client.calls.lock(),
1952 &[ModelId::from("weak"), "strong".into()]
1953 );
1954 assert_eq!(
1955 response
1956 .llm_response
1957 .as_agg()
1958 .map(|response| response.model.as_deref()),
1959 Some(Some("strong"))
1960 );
1961 assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
1962
1963 let (client, result) = run_candidates(FirstOutcome::ContentPolicy).await;
1965 let (_, response) = result?;
1966 assert_eq!(
1967 &*client.calls.lock(),
1968 &[ModelId::from("weak"), "strong".into()]
1969 );
1970 assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
1971
1972 let (client, result) = run_candidates(FirstOutcome::Unauthorized).await;
1974 assert!(matches!(
1975 result,
1976 Err(LibsyError::ClientCall {
1977 source: LlmClientError::UpstreamHttp {
1978 status: StatusCode::UNAUTHORIZED,
1979 ..
1980 },
1981 ..
1982 })
1983 ));
1984 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
1985 Ok(())
1986 }
1987
1988 #[tokio::test]
1989 async fn retry_budget_is_exhausted_before_falling_through() -> Result<()> {
1990 let server = MockServer::start().await;
1991 let calls = Arc::new(Mutex::new(Vec::new()));
1992 let observed_calls = Arc::clone(&calls);
1993 Mock::given(method("POST"))
1994 .respond_with(move |request: &wiremock::Request| {
1995 let body: serde_json::Value =
1996 serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
1997 let model = body["model"].as_str().unwrap_or_default().to_string();
1998 observed_calls.lock().push(model.clone());
1999 if model == "weak" {
2000 ResponseTemplate::new(503)
2001 .insert_header("retry-after", "0")
2002 .set_body_string("unavailable")
2003 } else {
2004 ResponseTemplate::new(200).set_body_json(serde_json::json!({
2005 "id": "answer",
2006 "model": "strong",
2007 "choices": [{
2008 "index": 0,
2009 "message": {"role": "assistant", "content": "ok"},
2010 "finish_reason": "stop"
2011 }],
2012 "usage": {}
2013 }))
2014 }
2015 })
2016 .mount(&server)
2017 .await;
2018
2019 let backend = || {
2020 Backend::OpenAiChat(HttpBackendConfig {
2021 base_url: format!("{}/v1", server.uri()),
2022 api_key: None,
2023 forward_auth: false,
2024 extra_headers: BTreeMap::new(),
2025 extra_body: BTreeMap::new(),
2026 reasoning_effort: None,
2027 max_retries: 2,
2028 timeout: None,
2029 })
2030 };
2031 let client = Arc::new(
2032 TranslatingLlmClient::new(&[
2033 ModelConfig::new("weak", backend(), None),
2034 ModelConfig::new("strong", backend(), None),
2035 ])
2036 .map_err(|error| LibsyError::external("building test client", error))?,
2037 );
2038 let algorithm = Arc::new(CandidateAlgorithm {});
2039 run(
2040 algorithm,
2041 ClientRouter::single(client),
2042 request(),
2043 to_category_map(&["weak", "strong"]),
2044 None,
2045 )
2046 .await?;
2047
2048 assert_eq!(&*calls.lock(), &["weak", "weak", "weak", "strong"]);
2049 Ok(())
2050 }
2051
2052 #[tokio::test]
2053 async fn streams_are_outside_the_candidate_fallback_boundary() -> Result<()> {
2054 let (client, result) = run_candidates(FirstOutcome::StreamSuccess).await;
2056 let (_, response) = result?;
2057 assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));
2058 let aggregate = response
2059 .llm_response
2060 .into_agg()
2061 .await
2062 .map_err(|source| LibsyError::client_call("weak", source))?;
2063 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2064 assert_eq!(completion_text(&aggregate), "streamed");
2065
2066 let (client, result) = run_candidates(FirstOutcome::MidStreamError).await;
2068 let (_, response) = result?;
2069 let error = response
2070 .llm_response
2071 .into_agg()
2072 .await
2073 .expect_err("the stream should fail during aggregation");
2074 assert!(error.to_string().contains("stream failed"));
2075 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2076 Ok(())
2077 }
2078
2079 const OVERFLOW_SSE: &str = "data: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.\",\"object\":\"error\",\"param\":null,\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n";
2082
2083 const STRONG_SSE: &str = "data: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
2085
2086 async fn run_streaming_candidates(
2090 weak_sse: &'static str,
2091 ) -> (
2092 MockServer,
2093 Arc<Mutex<Vec<String>>>,
2094 Result<(ModelId, Response)>,
2095 ) {
2096 let server = MockServer::start().await;
2097 let calls = Arc::new(Mutex::new(Vec::new()));
2098 let observed_calls = Arc::clone(&calls);
2099 Mock::given(method("POST"))
2100 .respond_with(move |request: &wiremock::Request| {
2101 let body: serde_json::Value =
2102 serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
2103 let model = body["model"].as_str().unwrap_or_default().to_string();
2104 observed_calls.lock().push(model.clone());
2105 let sse = if model == "weak" {
2106 weak_sse
2107 } else {
2108 STRONG_SSE
2109 };
2110 ResponseTemplate::new(200)
2111 .insert_header("content-type", "text/event-stream")
2112 .set_body_string(sse)
2113 })
2114 .mount(&server)
2115 .await;
2116
2117 let backend = || {
2118 Backend::OpenAiChat(HttpBackendConfig {
2119 base_url: format!("{}/v1", server.uri()),
2120 api_key: None,
2121 forward_auth: false,
2122 extra_headers: BTreeMap::new(),
2123 extra_body: BTreeMap::new(),
2124 reasoning_effort: None,
2125 max_retries: 0,
2126 timeout: None,
2127 })
2128 };
2129 let client = Arc::new(
2130 TranslatingLlmClient::new(&[
2131 ModelConfig::new("weak", backend(), None),
2132 ModelConfig::new("strong", backend(), None),
2133 ])
2134 .expect("building test client"),
2135 );
2136 let algorithm = Arc::new(CandidateAlgorithm {});
2137 let mut llm_request = text_request(Some("auto".to_string()), "hello".to_string());
2138 llm_request.stream = true;
2139 let request = Request {
2140 llm_request,
2141 raw_request: None,
2142 metadata: None,
2143 };
2144 let result = run(
2145 algorithm,
2146 ClientRouter::single(client),
2147 request,
2148 to_category_map(&["weak", "strong"]),
2149 None,
2150 )
2151 .await;
2152 (server, calls, result)
2153 }
2154
2155 #[tokio::test]
2156 async fn first_stream_event_overflow_falls_back_to_the_next_candidate() -> Result<()> {
2157 let (_server, calls, result) = run_streaming_candidates(OVERFLOW_SSE).await;
2160 let (_, response) = result?;
2161 assert_eq!(&*calls.lock(), &["weak", "strong"]);
2162 assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2163 let aggregate = response
2164 .llm_response
2165 .into_agg()
2166 .await
2167 .map_err(|source| LibsyError::client_call("strong", source))?;
2168 assert_eq!(completion_text(&aggregate), "ok");
2169
2170 let (_server, calls, result) = run_streaming_candidates(
2173 "data: {\"error\":{\"code\":500,\"message\":\"engine exploded\",\"object\":\"error\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
2174 )
2175 .await;
2176 let (_, response) = result?;
2177 assert_eq!(&*calls.lock(), &["weak"]);
2178 let error = response
2179 .llm_response
2180 .into_agg()
2181 .await
2182 .expect_err("the in-band error should surface to the consumer");
2183 assert!(error.to_string().contains("engine exploded"));
2184
2185 let (_server, calls, result) = run_streaming_candidates(
2188 "data: {\"id\":\"turn\",\"model\":\"weak\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"partial\"},\"finish_reason\":null}]}\n\ndata: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens).\",\"object\":\"error\",\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
2189 )
2190 .await;
2191 let (_, response) = result?;
2192 assert_eq!(&*calls.lock(), &["weak"]);
2193 let error = response
2194 .llm_response
2195 .into_agg()
2196 .await
2197 .expect_err("the mid-stream overflow should surface to the consumer");
2198 assert!(
2199 error
2200 .to_string()
2201 .contains("exceeds the maximum allowed length")
2202 );
2203
2204 let (_server, calls, result) = run_streaming_candidates("data: [DONE]\n\n").await;
2207 let (_, response) = result?;
2208 assert_eq!(&*calls.lock(), &["weak"]);
2209 let aggregate = response
2210 .llm_response
2211 .into_agg()
2212 .await
2213 .map_err(|source| LibsyError::client_call("weak", source))?;
2214 assert_eq!(completion_text(&aggregate), "");
2215 Ok(())
2216 }
2217}