1mod capabilities;
7pub mod config;
8mod metrics;
9mod observability;
10mod redaction;
11mod response;
12mod routing_log;
13mod shutdown;
14mod sse;
15mod stats;
16mod usage_metrics;
17
18use std::collections::BTreeMap;
19use std::error::Error;
20use std::fmt::{Display, Formatter};
21use std::future::Future;
22use std::io::IsTerminal;
23use std::net::SocketAddr;
24use std::path::PathBuf;
25use std::sync::Arc;
26use std::time::{Duration, Instant};
27
28use axum::body::Body;
29use axum::extract::rejection::{JsonRejection, QueryRejection};
30use axum::extract::{DefaultBodyLimit, Query, Request as HttpRequest, State};
31use axum::http::header::CONTENT_TYPE;
32use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
33use axum::middleware::Next;
34use axum::response::{IntoResponse, Response};
35use axum::routing::{get, post};
36use axum::{Extension, Json, Router};
37use axum_server::tls_rustls::RustlsConfig;
38use libsy::{LibsyError, RoutingOutcome};
39use parking_lot::Mutex;
40use serde::{Deserialize, Serialize};
41use serde_json::{Value, json};
42use switchyard_llm_client::{AuxiliaryOperation, RunObservation, RunObserver};
43use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};
44use switchyard_runner::{
45 CallerAuthKind, DecisionTarget, ModelCapabilities, Route, RunOutput, Runner, RunnerError,
46};
47use tokio::net::{TcpListener, TcpSocket};
48use tokio::task;
49use tracing::{Instrument, Level};
50
51use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response};
52
53use crate::response::into_http_response;
54use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env};
55
56pub use observability::{flush_observability, initialize_observability};
57
58pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;
60
61pub const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
63
64pub const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024;
66
67const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model";
68const FORWARDED_UPSTREAM_HEADERS: &[&str] = &[
69 "baggage",
70 "openai-processing-ms",
71 "request-id",
73 "traceparent",
74 "tracestate",
75 "x-request-id",
76];
77const FORWARDED_UPSTREAM_HEADER_PREFIXES: &[&str] =
78 &["anthropic-ratelimit-", "x-ratelimit-", "x-upstream-"];
79const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512;
80
81fn should_forward_upstream_header(name: &HeaderName) -> bool {
83 let name = name.as_str();
84 FORWARDED_UPSTREAM_HEADERS.contains(&name)
85 || FORWARDED_UPSTREAM_HEADER_PREFIXES
86 .iter()
87 .any(|prefix| name.starts_with(prefix))
88}
89const CLIENT_CLOSED_REQUEST: u16 = 499;
92const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt");
93
94#[derive(Debug)]
96pub struct ServerError {
97 message: String,
98}
99
100impl ServerError {
101 pub fn new(message: impl Into<String>) -> Self {
103 Self {
104 message: message.into(),
105 }
106 }
107}
108
109impl Display for ServerError {
110 fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
111 formatter.write_str(&self.message)
112 }
113}
114
115impl Error for ServerError {}
116
117impl From<RunnerError> for ServerError {
118 fn from(error: RunnerError) -> Self {
119 let mut message = error.to_string();
120 let mut source = error.source();
121 while let Some(error) = source {
122 message.push_str(": ");
123 message.push_str(&error.to_string());
124 source = error.source();
125 }
126 Self::new(message)
127 }
128}
129
130pub type ServerResult<T> = std::result::Result<T, ServerError>;
132
133#[derive(Serialize)]
135struct DecisionResponse {
136 selected: DecisionTargetResponse,
137 fallbacks: Vec<DecisionTargetResponse>,
138 #[serde(skip_serializing_if = "Option::is_none")]
139 response: Option<Value>,
140}
141
142#[derive(Serialize)]
144struct DecisionTargetResponse {
145 target: String,
146 model: ModelId,
147 llm_client: DecisionLlmClientResponse,
148 extra_body: BTreeMap<String, Value>,
149}
150
151#[derive(Serialize)]
153struct DecisionLlmClientResponse {
154 format: WireFormat,
155 base_url: String,
156}
157
158#[derive(Clone)]
160pub struct ServerState {
161 runner: Arc<Runner>,
162 redactor: Arc<redaction::Redactor>,
163 fallback_http: reqwest::Client,
164 metrics: prometheus::Registry,
165 stats: StatsAccumulator,
166 routing_log: Option<SharedRoutingLog>,
167 track_cache_eligibility: bool,
168}
169
170#[derive(Clone)]
171struct SharedRoutingLog {
172 writer: Arc<Mutex<routing_log::RoutingLog>>,
173 path: PathBuf,
174}
175
176impl SharedRoutingLog {
177 fn new(path: PathBuf) -> ServerResult<Self> {
178 Ok(Self {
179 writer: Arc::new(Mutex::new(routing_log::RoutingLog::new(path.clone())?)),
180 path,
181 })
182 }
183
184 fn append(
185 &self,
186 context: routing_log::RoutingLogContext,
187 model: &str,
188 tier: Option<&str>,
189 usage: &Usage,
190 ) {
191 if let Err(error) = self.writer.lock().append(context, model, tier, usage) {
192 tracing::warn!(path = %self.path.display(), %error, "routing log append failed");
193 }
194 }
195
196 fn snapshot_session(
197 &self,
198 session_id: &str,
199 ) -> std::io::Result<Option<routing_log::SessionStatsSnapshot>> {
200 routing_log::snapshot(&self.path, session_id)
201 }
202}
203
204impl ServerState {
205 pub fn from_runner(runner: Runner) -> ServerResult<Self> {
207 let metrics = metrics::registry().map_err(ServerError::new)?;
208 let fallback_http = reqwest::Client::builder()
209 .redirect(reqwest::redirect::Policy::none())
210 .build()
211 .map_err(|error| ServerError::new(error.to_string()))?;
212 let stats = StatsAccumulator::new(
213 metrics.clone(),
214 runner.models().map(|model| model.algorithm),
215 );
216 let redactor = redaction::Redactor::new(runner.provider_api_keys());
217 Ok(Self {
218 redactor: Arc::new(redactor),
219 runner: Arc::new(runner),
220 fallback_http,
221 metrics,
222 stats,
223 routing_log: None,
224 track_cache_eligibility: tracking_enabled_from_env(),
225 })
226 }
227
228 pub fn with_routing_log(mut self, path: impl Into<PathBuf>) -> ServerResult<Self> {
230 self.routing_log = Some(SharedRoutingLog::new(path.into())?);
231 Ok(self)
232 }
233
234 pub fn models(&self) -> impl Iterator<Item = &str> {
236 self.runner.models().map(|model| model.id.as_str())
237 }
238
239 pub fn caller_auth_kind(&self, model: &str) -> Option<&'static str> {
241 self.runner
242 .route(model)
243 .and_then(|route| route.caller_auth())
244 .map(|kind| kind.as_str())
245 }
246
247 fn route_for_model(&self, model: &str) -> Option<&Route> {
248 self.runner.route(model)
249 }
250
251 fn decision_response(
252 &self,
253 route_model: &ModelId,
254 outcome: &RoutingOutcome,
255 response: Option<Value>,
256 ) -> Option<DecisionResponse> {
257 let description = self.runner.describe_decision(route_model, outcome)?;
258 let convert = |target: DecisionTarget| DecisionTargetResponse {
259 target: target.target,
260 model: target.model,
261 llm_client: DecisionLlmClientResponse {
262 format: target.format,
263 base_url: target.base_url,
264 },
265 extra_body: target.extra_body,
266 };
267 Some(DecisionResponse {
268 selected: convert(description.selected),
269 fallbacks: description.fallbacks.into_iter().map(convert).collect(),
270 response,
271 })
272 }
273}
274#[derive(Clone, Debug)]
276pub struct ServerRunOptions {
277 pub addr: SocketAddr,
279 pub backlog: u32,
281 pub dry_run: bool,
283 pub shutdown_timeout: Duration,
285 pub tls: Option<TlsOptions>,
287}
288
289#[derive(Clone, Debug)]
291pub struct TlsOptions {
292 pub cert: PathBuf,
294 pub key: PathBuf,
296}
297
298impl ServerRunOptions {
299 fn is_tls(&self) -> bool {
300 self.tls.is_some()
301 }
302}
303
304pub async fn run_server(state: ServerState, options: ServerRunOptions) -> ServerResult<()> {
306 if options.dry_run {
307 println!("{}", dry_run_summary(&state));
308 return Ok(());
309 }
310
311 let server = BoundServer::bind(state, options)?;
312 println!("{}", server.startup_banner(std::io::stdout().is_terminal()));
313 server.serve(shutdown::signal()).await
314}
315
316pub struct BoundServer {
318 listener: TcpListener,
319 router: Router,
320 options: ServerRunOptions,
321 state: ServerState,
322}
323
324impl BoundServer {
325 pub fn bind(state: ServerState, options: ServerRunOptions) -> ServerResult<Self> {
327 let listener = bind_tcp_listener(options.addr, options.backlog)?;
328 let addr = listener.local_addr().map_err(server_io_error)?;
329 Ok(Self {
330 listener,
331 router: build_switchyard_router(state.clone()),
332 options: ServerRunOptions { addr, ..options },
333 state,
334 })
335 }
336
337 pub fn local_addr(&self) -> SocketAddr {
339 self.options.addr
340 }
341
342 pub async fn serve(
344 self,
345 shutdown: impl Future<Output = ()> + Send + 'static,
346 ) -> ServerResult<()> {
347 let shutdown_timeout = self.options.shutdown_timeout;
348 if let Some(tls) = self.options.tls {
349 serve_tls(self.listener, self.router, tls, shutdown_timeout, shutdown).await
350 } else {
351 serve(self.listener, self.router, shutdown_timeout, shutdown).await
352 }
353 }
354
355 fn startup_banner(&self, color: bool) -> String {
356 startup_banner(&self.options, &self.state, color)
357 }
358}
359
360async fn serve_tls(
361 listener: TcpListener,
362 router: Router,
363 tls: TlsOptions,
364 shutdown_timeout: Duration,
365 shutdown: impl Future<Output = ()> + Send + 'static,
366) -> ServerResult<()> {
367 if let Err(error) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
368 tracing::debug!(?error, "TLS crypto provider was already installed");
369 }
370
371 let config = RustlsConfig::from_pem_file(tls.cert, tls.key)
372 .await
373 .map_err(server_io_error)?;
374 let std_listener = listener.into_std().map_err(server_io_error)?;
375 let server = axum_server::from_tcp_rustls(std_listener, config).map_err(server_io_error)?;
376 let handle = axum_server::Handle::new();
377 let server = server
378 .handle(handle.clone())
379 .serve(router.into_make_service());
380 serve_until_shutdown(server, handle, shutdown_timeout, shutdown).await
381}
382
383async fn serve(
384 listener: TcpListener,
385 router: Router,
386 shutdown_timeout: Duration,
387 shutdown: impl Future<Output = ()> + Send + 'static,
388) -> ServerResult<()> {
389 let std_listener = listener.into_std().map_err(server_io_error)?;
390 let server = axum_server::from_tcp(std_listener).map_err(server_io_error)?;
391 let handle = axum_server::Handle::new();
392 let server = server
393 .handle(handle.clone())
394 .serve(router.into_make_service());
395 serve_until_shutdown(server, handle, shutdown_timeout, shutdown).await
396}
397
398async fn serve_until_shutdown(
400 server: impl Future<Output = std::io::Result<()>>,
401 handle: axum_server::Handle<SocketAddr>,
402 timeout: Duration,
403 shutdown: impl Future<Output = ()> + Send + 'static,
404) -> ServerResult<()> {
405 tokio::pin!(server);
406 tokio::select! {
407 result = &mut server => result.map_err(server_io_error),
408 _ = shutdown => {
409 tracing::info!(
410 ?timeout,
411 "shutdown signal received; draining active requests"
412 );
413 handle.graceful_shutdown(Some(timeout));
414 server.await.map_err(server_io_error)
415 }
416 }
417}
418
419#[derive(Clone, Copy)]
421struct RequestStart(Instant);
422
423const CLASSIFIER_TIER: &str = "classifier";
425
426fn stats_observer(
434 stats: StatsAccumulator,
435 classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>,
436) -> RunObserver {
437 Arc::new(move |observation| match observation {
438 RunObservation::Outcome(_) => {}
439 RunObservation::AnswerCall(call) => {
440 let latency_ms = call.duration.as_secs_f64() * 1_000.0;
441 if call.is_success {
442 stats.record_success(&call.selected_model, latency_ms);
443 } else {
444 stats.record_error(&call.selected_model);
445 }
446 }
447 RunObservation::LlmCall(call) => {
448 let latency_ms = call.duration.as_secs_f64() * 1_000.0;
449 if call.is_success {
450 if let (Some((log, context)), Some(usage)) =
451 (classifier_log.as_ref(), call.usage.as_ref())
452 {
453 log.append(
454 context.clone(),
455 &call.selected_model,
456 Some(CLASSIFIER_TIER),
457 usage,
458 );
459 }
460 stats.record_classifier_success(
461 call.selected_model,
462 call.usage.as_ref().map(usage_metrics::token_usage),
463 latency_ms,
464 );
465 } else {
466 stats.record_classifier_error(call.selected_model);
467 }
468 }
469 RunObservation::RoutingOverhead(duration) => {
470 stats.record_routing_overhead(duration.as_secs_f64() * 1_000.0);
471 }
472 })
473}
474
475async fn stamp_request_start(mut request: HttpRequest, next: Next) -> Response {
479 request
480 .extensions_mut()
481 .insert(RequestStart(Instant::now()));
482 next.run(request).await
483}
484
485pub fn build_llm_router(state: ServerState) -> Router {
491 finish_router(primary_llm_routes(), state)
492}
493
494pub fn build_switchyard_router(state: ServerState) -> Router {
496 let mut router = primary_llm_routes()
497 .route("/v1/decision", post(decision))
498 .route("/v1/messages/count_tokens", post(anthropic_count_tokens))
499 .route(
500 "/v1/responses/input_tokens",
501 post(openai_responses_input_tokens),
502 )
503 .route("/v1/responses/compact", post(openai_responses_compact))
504 .route("/v1/models", get(models))
505 .route("/v1/stats", get(get_stats))
506 .route("/v1/stats/reset", post(reset_stats))
507 .route("/metrics", get(prometheus_metrics))
508 .route("/health", get(health));
509 if state.routing_log.is_some() {
510 router = router.route("/v1/routing/session-stats", get(get_session_stats));
511 }
512 finish_router(router.fallback(proxy_unmatched), state)
513}
514
515fn primary_llm_routes() -> Router<ServerState> {
517 Router::new()
518 .route("/v1/chat/completions", post(openai_chat_completions))
519 .route("/v1/messages", post(anthropic_messages))
520 .route("/v1/responses", post(openai_responses))
521}
522
523fn finish_router(router: Router<ServerState>, state: ServerState) -> Router {
525 router
526 .layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))
527 .layer(axum::middleware::from_fn_with_state(
528 state.clone(),
529 redaction::redact_response,
530 ))
531 .layer(axum::middleware::from_fn(stamp_request_start))
533 .with_state(state)
534}
535
536fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> ServerResult<TcpListener> {
537 let socket = if addr.is_ipv4() {
538 TcpSocket::new_v4()
539 } else {
540 TcpSocket::new_v6()
541 }
542 .map_err(server_io_error)?;
543
544 socket.set_reuseaddr(true).map_err(server_io_error)?;
545 socket.bind(addr).map_err(server_io_error)?;
546 socket.listen(backlog).map_err(server_io_error)
547}
548
549fn server_io_error(error: std::io::Error) -> ServerError {
550 ServerError::new(error.to_string())
551}
552
553async fn openai_chat_completions(
554 State(state): State<ServerState>,
555 Extension(started): Extension<RequestStart>,
556 headers: HeaderMap,
557 body: std::result::Result<Json<Value>, JsonRejection>,
558) -> Response {
559 handle_endpoint(state, started, headers, body, WireFormat::OpenAiChat).await
560}
561
562async fn anthropic_messages(
563 State(state): State<ServerState>,
564 Extension(started): Extension<RequestStart>,
565 headers: HeaderMap,
566 body: std::result::Result<Json<Value>, JsonRejection>,
567) -> Response {
568 handle_endpoint(state, started, headers, body, WireFormat::AnthropicMessages).await
569}
570
571async fn openai_responses(
572 State(state): State<ServerState>,
573 Extension(started): Extension<RequestStart>,
574 headers: HeaderMap,
575 body: std::result::Result<Json<Value>, JsonRejection>,
576) -> Response {
577 handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await
578}
579
580async fn proxy_unmatched(State(state): State<ServerState>, request: HttpRequest) -> Response {
582 let Some(base_url) = state.runner.fallback_base_url() else {
583 return not_found().await;
584 };
585 let (mut parts, body) = request.into_parts();
586 let path_and_query = parts
587 .uri
588 .path_and_query()
589 .map_or_else(|| parts.uri.path().to_string(), ToString::to_string);
590 strip_hop_by_hop_headers(&mut parts.headers);
591 parts.headers.remove(axum::http::header::HOST);
592 let url = match fallback_url(base_url, &path_and_query) {
593 Ok(url) => url,
594 Err(error) => {
595 return error_response(
596 StatusCode::BAD_GATEWAY,
597 error,
598 "upstream_error",
599 "upstream_error",
600 );
601 }
602 };
603 let body = reqwest::Body::wrap_stream(body.into_data_stream());
604 match state
605 .fallback_http
606 .request(parts.method, url)
607 .headers(parts.headers)
608 .body(body)
609 .send()
610 .await
611 {
612 Ok(response) => {
613 let response: http::Response<reqwest::Body> = response.into();
614 let (mut parts, body) = response.into_parts();
615 strip_hop_by_hop_headers(&mut parts.headers);
616 Response::from_parts(parts, Body::new(body))
617 }
618 Err(error) => error_response(
619 StatusCode::BAD_GATEWAY,
620 error.without_url().to_string(),
621 "upstream_error",
622 "upstream_error",
623 ),
624 }
625}
626
627fn fallback_url(base_url: &str, path_and_query: &str) -> Result<reqwest::Url, String> {
628 let mut url = reqwest::Url::parse(base_url.trim())
629 .map_err(|error| format!("invalid fallback base URL: {error}"))?;
630 let base_query = url.query().map(str::to_owned);
631 url.set_query(None);
632 url.set_fragment(None);
633
634 let base_path = url.path().trim_end_matches('/');
635 let root_path = [
636 "/v1/chat/completions",
637 "/v1/responses",
638 "/v1/messages",
639 "/v1",
640 ]
641 .iter()
642 .find_map(|suffix| base_path.strip_suffix(suffix))
643 .unwrap_or(base_path);
644 let (request_path, request_query) = path_and_query
645 .split_once('?')
646 .map_or((path_and_query, None), |(path, query)| (path, Some(query)));
647 let request_path = request_path.strip_prefix('/').unwrap_or(request_path);
648 let target_path = if root_path.is_empty() {
649 format!("/{request_path}")
650 } else if request_path.is_empty() {
651 format!("{root_path}/")
652 } else {
653 format!("{root_path}/{request_path}")
654 };
655 url.set_path(&target_path);
656
657 let query = match (base_query, request_query) {
658 (Some(base_query), Some(request_query)) => Some(format!("{base_query}&{request_query}")),
659 (Some(base_query), None) => Some(base_query),
660 (None, Some(request_query)) => Some(request_query.to_owned()),
661 (None, None) => None,
662 };
663 url.set_query(query.as_deref());
664 Ok(url)
665}
666
667fn strip_hop_by_hop_headers(headers: &mut HeaderMap) {
668 let connection_headers = headers
669 .get_all(axum::http::header::CONNECTION)
670 .iter()
671 .filter_map(|value| value.to_str().ok())
672 .flat_map(|value| value.split(','))
673 .filter_map(|name| HeaderName::from_bytes(name.trim().as_bytes()).ok())
674 .collect::<Vec<_>>();
675 for name in connection_headers {
676 headers.remove(name);
677 }
678 for name in [
679 "connection",
680 "keep-alive",
681 "proxy-authenticate",
682 "proxy-authorization",
683 "proxy-connection",
684 "te",
685 "trailer",
686 "transfer-encoding",
687 "upgrade",
688 ] {
689 headers.remove(name);
690 }
691}
692
693#[derive(Deserialize)]
695#[serde(deny_unknown_fields)]
696struct DecisionEndpointRequest {
697 input_format: WireFormat,
698 request: Value,
699}
700
701async fn decision(
703 State(state): State<ServerState>,
704 headers: HeaderMap,
705 body: std::result::Result<Json<DecisionEndpointRequest>, JsonRejection>,
706) -> Response {
707 let body = match body {
708 Ok(Json(body)) if body.request.is_object() => body,
709 Ok(_) => {
710 return invalid_body_error(StatusCode::BAD_REQUEST, "request must be a JSON object");
711 }
712 Err(error) => {
713 return invalid_body_error(
714 error.status(),
715 format!("Request body must be valid JSON: {error}"),
716 );
717 }
718 };
719 let input_format = body.input_format;
720 let (route, request) = match resolve_route(
721 &state,
722 metadata_from_headers(headers),
723 body.request,
724 input_format,
725 ) {
726 Ok(resolved) => resolved,
727 Err(response) => return response,
728 };
729 let route_model = request
730 .llm_request
731 .model
732 .as_deref()
733 .map(ModelId::from)
734 .unwrap_or_default();
735
736 let mut outcome = match route.decide(request).await {
737 Ok(outcome) => outcome,
738 Err(error) => return runner_error(error),
739 };
740 let response = match outcome.response.take() {
741 Some(response) => {
742 let aggregate = match response.llm_response.into_agg().await {
743 Ok(aggregate) => aggregate,
744 Err(error) => return client_error(&error),
745 };
746 match encode_aggregated_response(
750 &aggregate,
751 input_format,
752 outcome.selected_model_id().ok().map(ModelId::as_str),
753 ) {
754 Ok(response) => Some(response),
755 Err(error) => return server_error(error.to_string()),
756 }
757 }
758 None => None,
759 };
760 match state.decision_response(&route_model, &outcome, response) {
761 Some(response) => Json(response).into_response(),
762 None => {
763 server_error("routing outcome contains a model with no callable target configuration")
764 }
765 }
766}
767
768async fn anthropic_count_tokens(
770 State(state): State<ServerState>,
771 headers: HeaderMap,
772 body: std::result::Result<Json<Value>, JsonRejection>,
773) -> Response {
774 let body = match llm_json_body(body) {
775 Ok(body) => body,
776 Err((status, message)) => {
777 return anthropic_error_response(invalid_body_error(status, message));
778 }
779 };
780 let (route, request) = match resolve_route(
781 &state,
782 metadata_from_headers(headers),
783 body,
784 WireFormat::AnthropicMessages,
785 ) {
786 Ok(resolved) => resolved,
787 Err(response) => return anthropic_error_response(response),
788 };
789 anthropic_error_response(
790 match route
791 .call_auxiliary(request, AuxiliaryOperation::AnthropicCountTokens)
792 .await
793 {
794 Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
795 Err(RunnerError::AuxiliaryUnsupported) => error_response(
796 StatusCode::BAD_REQUEST,
797 "route has no Anthropic target for token counting",
798 "invalid_request_error",
799 "count_tokens_unsupported",
800 ),
801 Err(RunnerError::Client(error)) => client_error(&error),
802 Err(error) => server_error(error.to_string()),
803 },
804 )
805}
806
807async fn openai_responses_input_tokens(
808 State(state): State<ServerState>,
809 headers: HeaderMap,
810 body: std::result::Result<Json<Value>, JsonRejection>,
811) -> Response {
812 openai_responses_auxiliary(
813 state,
814 headers,
815 body,
816 AuxiliaryOperation::ResponsesInputTokens,
817 )
818 .await
819}
820
821async fn openai_responses_compact(
822 State(state): State<ServerState>,
823 headers: HeaderMap,
824 body: std::result::Result<Json<Value>, JsonRejection>,
825) -> Response {
826 openai_responses_auxiliary(state, headers, body, AuxiliaryOperation::ResponsesCompact).await
827}
828
829async fn openai_responses_auxiliary(
831 state: ServerState,
832 headers: HeaderMap,
833 body: std::result::Result<Json<Value>, JsonRejection>,
834 operation: AuxiliaryOperation,
835) -> Response {
836 let body = match llm_json_body(body) {
837 Ok(body) => body,
838 Err((status, message)) => {
839 return render_error_response(
840 invalid_body_error(status, message),
841 WireFormat::OpenAiResponses,
842 );
843 }
844 };
845 let (route, request) = match resolve_route(
846 &state,
847 metadata_from_headers(headers),
848 body,
849 WireFormat::OpenAiResponses,
850 ) {
851 Ok(resolved) => resolved,
852 Err(response) => return render_error_response(response, WireFormat::OpenAiResponses),
853 };
854 let result = route.call_auxiliary(request, operation).await;
855 render_error_response(
856 match result {
857 Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
858 Err(RunnerError::AuxiliaryUnsupported) => error_response(
859 StatusCode::BAD_REQUEST,
860 "route has no OpenAI Responses target",
861 "invalid_request_error",
862 "responses_auxiliary_unsupported",
863 ),
864 Err(RunnerError::Client(error)) => client_error(&error),
865 Err(error) => server_error(error.to_string()),
866 },
867 WireFormat::OpenAiResponses,
868 )
869}
870
871async fn handle_endpoint(
872 state: ServerState,
873 started: RequestStart,
874 headers: HeaderMap,
875 body: std::result::Result<Json<Value>, JsonRejection>,
876 wire_format: WireFormat,
877) -> Response {
878 let span = observability::request_span(&headers);
879 handle_endpoint_inner(state, started, headers, body, wire_format)
880 .instrument(span)
881 .await
882}
883
884async fn handle_endpoint_inner(
885 state: ServerState,
886 started: RequestStart,
887 headers: HeaderMap,
888 body: std::result::Result<Json<Value>, JsonRejection>,
889 wire_format: WireFormat,
890) -> Response {
891 let metadata = metadata_from_headers(headers);
892 let routing_log_context = state
893 .routing_log
894 .as_ref()
895 .map(|_| routing_log::RoutingLogContext::from_metadata(&metadata));
896 let request_log = RequestLogContext {
897 started: started.0,
898 wire_format,
899 requested_model: body
900 .as_ref()
901 .ok()
902 .and_then(|body| body.0.get("model"))
903 .and_then(Value::as_str)
904 .map(str::to_string),
905 streaming: body
906 .as_ref()
907 .ok()
908 .and_then(|body| body.0.get("stream"))
909 .and_then(Value::as_bool)
910 .unwrap_or(false),
911 session_id: metadata.session_id.clone(),
912 correlation_id: metadata.correlation_id.clone(),
913 };
914
915 let mut request_log = RequestLogGuard(Some(request_log));
920
921 let response = match llm_json_body(body) {
922 Ok(body) => {
923 handle_llm_request(
924 state,
925 started,
926 metadata,
927 body,
928 wire_format,
929 routing_log_context,
930 )
931 .await
932 }
933 Err((status, message)) => invalid_body_error(status, message),
934 };
935 let response = render_error_response(response, wire_format);
936 metrics::record_client_response(response.status().as_u16());
937 request_log.emit(&response);
938 response
939}
940
941fn llm_json_body(
942 body: std::result::Result<Json<Value>, JsonRejection>,
943) -> std::result::Result<Value, (StatusCode, String)> {
944 match body {
945 Ok(Json(value)) if value.is_object() => Ok(value),
946 Ok(_) => Err((
947 StatusCode::BAD_REQUEST,
948 "Request body must be a JSON object".to_string(),
949 )),
950 Err(error) => Err((
951 error.status(),
952 format!("Request body must be valid JSON: {error}"),
953 )),
954 }
955}
956
957#[allow(clippy::type_complexity, clippy::result_large_err)]
964fn resolve_route(
965 state: &ServerState,
966 metadata: Metadata,
967 mut body: Value,
968 wire_format: WireFormat,
969) -> std::result::Result<(&Route, Request), Response> {
970 if let Some(metadata) = body.get_mut("metadata").and_then(Value::as_object_mut) {
973 metadata.remove(switchyard_translation::util::SWITCHYARD_METADATA_KEY);
974 }
975 let llm_request = decode_request(wire_format, &body)
976 .map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?;
977 let requested_model = llm_request
978 .model
979 .clone()
980 .filter(|model| !model.trim().is_empty())
981 .ok_or_else(|| {
982 error_response(
983 StatusCode::BAD_REQUEST,
984 "request body must include a non-empty string `model`",
985 "invalid_request_error",
986 "invalid_request_error",
987 )
988 })?;
989 let route = state.route_for_model(&requested_model).ok_or_else(|| {
990 error_response(
991 StatusCode::NOT_FOUND,
992 format!("No route registered for model {requested_model}"),
993 "model_not_found",
994 "model_not_found",
995 )
996 })?;
997 if let Err(RunnerError::IncompatibleCallerFormat(caller_auth)) =
998 route.check_caller_format(wire_format)
999 {
1000 let (provider, expected_endpoint) = match caller_auth {
1001 CallerAuthKind::Anthropic => ("Anthropic", "/v1/messages"),
1002 CallerAuthKind::OpenAi => ("OpenAI", "/v1/chat/completions or /v1/responses"),
1003 };
1004 return Err(error_response(
1005 StatusCode::BAD_REQUEST,
1006 format!(
1007 "route {requested_model} forwards an {provider} login; call it through {expected_endpoint}",
1008 ),
1009 "invalid_request_error",
1010 "invalid_request_error",
1011 ));
1012 }
1013 if let Some(capability) =
1014 capabilities::unsupported_capability(route.capabilities(), &llm_request, &body)
1015 {
1016 return Err(error_response(
1017 StatusCode::BAD_REQUEST,
1018 format!(
1019 "route {requested_model} declares {capability} = false; remove the unsupported input or select another route"
1020 ),
1021 "invalid_request_error",
1022 "unsupported_capability",
1023 ));
1024 }
1025 let request = Request {
1026 llm_request,
1027 raw_request: Some(body),
1028 metadata: Some(metadata),
1029 };
1030 Ok((route, request))
1031}
1032
1033async fn handle_llm_request(
1035 state: ServerState,
1036 started: RequestStart,
1037 metadata: Metadata,
1038 body: Value,
1039 wire_format: WireFormat,
1040 routing_log_context: Option<routing_log::RoutingLogContext>,
1041) -> Response {
1042 let cache_probe = state.track_cache_eligibility.then(|| prefix_probe(&body));
1043 let (route, request) = match resolve_route(&state, metadata, body, wire_format) {
1044 Ok(resolved) => resolved,
1045 Err(response) => return response,
1046 };
1047 let routing_log_context = routing_log_context.map(|context| {
1048 context.with_route(
1049 request.llm_request.model.as_deref().unwrap_or_default(),
1050 route.algorithm_name(),
1051 )
1052 });
1053 let request_extensions = request.llm_request.extensions.clone();
1055 let observer = stats_observer(
1056 state.stats.clone(),
1057 state.routing_log.clone().zip(routing_log_context.clone()),
1058 );
1059
1060 let output = match route.execute(request, Some(observer)).await {
1061 Ok(output) => output,
1062 Err(error) => {
1063 if let RunnerError::Algorithm(LibsyError::ClientCall {
1064 target,
1065 source:
1066 LlmClientError::ResponseStateLimitExceeded { .. }
1067 | LlmClientError::ResponseStateConflict,
1068 }) = &error
1069 {
1070 state.stats.record_response_error(target);
1071 }
1072 return runner_error(error);
1073 }
1074 };
1075 let RunOutput {
1076 selected_model,
1077 response,
1078 } = output;
1079 let served_model = response.served_model().cloned().or(Some(selected_model));
1082 let mut response = if let Some(served_model) = served_model.as_ref() {
1083 let cache_eligible = cache_probe
1084 .as_ref()
1085 .map(|probe| state.stats.prefix_eligibility(served_model, probe))
1086 .unwrap_or(0.0);
1087 usage_metrics::observe(
1088 response,
1089 served_model.as_str(),
1090 started.0,
1091 state.stats,
1092 cache_eligible,
1093 state.routing_log.zip(routing_log_context),
1094 )
1095 } else {
1096 response
1097 };
1098
1099 let upstream_headers = std::mem::take(&mut response.upstream_headers);
1100 let response_model = served_model.as_ref().map(ToString::to_string);
1101 let mut response = match into_http_response(
1102 response,
1103 wire_format,
1104 response_model,
1105 request_extensions,
1106 Arc::clone(&state.redactor),
1107 ) {
1108 Ok(response) => response,
1109 Err(error) => return server_error(error.to_string()),
1110 };
1111 let response_headers = response.headers_mut();
1114 for (name, value) in upstream_headers.iter() {
1115 if !should_forward_upstream_header(name) {
1116 continue;
1117 }
1118 response_headers.append(name.clone(), value.clone());
1119 }
1120 if let Some(served_model) = served_model.as_ref() {
1121 attach_routing_headers(&mut response, served_model.as_str());
1122 }
1123 response
1124}
1125
1126struct RequestLogGuard(Option<RequestLogContext>);
1131
1132impl RequestLogGuard {
1133 fn emit(&mut self, response: &Response) {
1135 if let Some(context) = self.0.take() {
1136 context.emit(response);
1137 }
1138 }
1139}
1140
1141impl Drop for RequestLogGuard {
1142 fn drop(&mut self) {
1143 if let Some(context) = self.0.take() {
1144 context.emit_cancelled();
1145 }
1146 }
1147}
1148
1149struct RequestLogContext {
1151 started: Instant,
1152 wire_format: WireFormat,
1153 requested_model: Option<String>,
1154 streaming: bool,
1155 session_id: Option<String>,
1156 correlation_id: Option<String>,
1157}
1158
1159#[derive(Clone)]
1161struct RequestLogError(String);
1162
1163impl RequestLogContext {
1164 fn emit(self, response: &Response) {
1165 let selected_model = response
1166 .headers()
1167 .get(HEADER_SELECTED_MODEL)
1168 .and_then(|value| value.to_str().ok())
1169 .unwrap_or("");
1170 let duration_ms = self.started.elapsed().as_secs_f64() * 1_000.0;
1171 let error = response
1172 .extensions()
1173 .get::<RequestLogError>()
1174 .map(|error| error.0.as_str())
1175 .unwrap_or("");
1176
1177 macro_rules! emit {
1178 ($level:expr, $message:literal) => {
1179 tracing::event!(
1180 target: "switchyard_server::request",
1181 $level,
1182 wire_format = %self.wire_format,
1183 status = response.status().as_u16(),
1184 requested_model = self.requested_model.as_deref().unwrap_or(""),
1185 selected_model,
1186 streaming = self.streaming,
1187 session_id = self.session_id.as_deref().unwrap_or(""),
1188 correlation_id = self.correlation_id.as_deref().unwrap_or(""),
1189 handling_duration_ms = duration_ms,
1190 error,
1191 $message
1192 )
1193 };
1194 }
1195
1196 match request_log_level(response.status()) {
1197 Level::ERROR => emit!(Level::ERROR, "LLM request failed"),
1198 Level::WARN => emit!(Level::WARN, "LLM request failed"),
1199 _ => emit!(Level::INFO, "LLM request handled"),
1200 }
1201 }
1202
1203 fn emit_cancelled(self) {
1206 metrics::record_client_disconnect();
1207 tracing::event!(
1208 target: "switchyard_server::request",
1209 Level::WARN,
1210 wire_format = %self.wire_format,
1211 status = CLIENT_CLOSED_REQUEST,
1212 requested_model = self.requested_model.as_deref().unwrap_or(""),
1213 selected_model = "",
1214 streaming = self.streaming,
1215 session_id = self.session_id.as_deref().unwrap_or(""),
1216 correlation_id = self.correlation_id.as_deref().unwrap_or(""),
1217 handling_duration_ms = self.started.elapsed().as_secs_f64() * 1_000.0,
1218 error = "client disconnected before a response was written",
1219 "LLM request cancelled"
1220 );
1221 }
1222}
1223
1224fn request_log_level(status: StatusCode) -> Level {
1225 if status.is_server_error() {
1226 Level::ERROR
1227 } else if status.is_success() {
1228 Level::INFO
1229 } else {
1230 Level::WARN
1231 }
1232}
1233
1234fn metadata_from_headers(headers: HeaderMap) -> Metadata {
1235 let mut metadata = Metadata::from_headers(&headers);
1236 metadata.http_headers = Some(headers);
1237 metadata
1238}
1239
1240fn attach_routing_headers(response: &mut Response, served_model: &str) {
1241 insert_routing_header(response, HEADER_SELECTED_MODEL, served_model);
1242}
1243
1244fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) {
1245 let Some(value) = sanitize_routing_header_value(value) else {
1246 return;
1247 };
1248 let Ok(value) = HeaderValue::from_str(&value) else {
1249 return;
1250 };
1251 response
1252 .headers_mut()
1253 .insert(HeaderName::from_static(name), value);
1254}
1255
1256fn sanitize_routing_header_value(value: &str) -> Option<String> {
1257 let value = value.split_whitespace().collect::<Vec<_>>().join(" ");
1258 (!value.is_empty()).then(|| value.chars().take(MAX_ROUTING_HEADER_VALUE_LEN).collect())
1259}
1260
1261fn algorithm_error(error: LibsyError) -> Response {
1262 let LibsyError::ClientCall { source, .. } = &error else {
1263 return server_error(error.to_string());
1264 };
1265 client_error(source)
1266}
1267
1268fn runner_error(error: RunnerError) -> Response {
1269 match error {
1270 RunnerError::Algorithm(error) => algorithm_error(error),
1271 RunnerError::Client(error) => client_error(&error),
1272 error => server_error(error.to_string()),
1273 }
1274}
1275
1276fn client_error(error: &LlmClientError) -> Response {
1277 match error {
1278 LlmClientError::ResponseStateLimitExceeded { .. } => error_response(
1279 StatusCode::SERVICE_UNAVAILABLE,
1280 error.to_string(),
1281 "server_error",
1282 "response_state_limit_exceeded",
1283 ),
1284 LlmClientError::ResponseStateConflict => error_response(
1285 StatusCode::CONFLICT,
1286 error.to_string(),
1287 "server_error",
1288 "response_state_conflict",
1289 ),
1290 LlmClientError::InvalidRequest { message }
1291 | LlmClientError::RequestTranslation(message) => error_response(
1292 StatusCode::BAD_REQUEST,
1293 message,
1294 "invalid_request_error",
1295 "invalid_request_error",
1296 ),
1297 LlmClientError::Configuration { message } => error_response(
1298 StatusCode::BAD_GATEWAY,
1299 message,
1300 "upstream_error",
1301 "upstream_configuration_error",
1302 ),
1303 LlmClientError::ContextWindowExceeded { message, .. } => error_response(
1304 StatusCode::BAD_REQUEST,
1305 message,
1306 "invalid_request_error",
1307 "context_length_exceeded",
1308 ),
1309 LlmClientError::UpstreamHttp { status, body } => upstream_error(*status, body),
1310 LlmClientError::Transport { source } | LlmClientError::InvalidResponse { source } => {
1311 error_response(
1312 StatusCode::BAD_GATEWAY,
1313 source.to_string(),
1314 "upstream_error",
1315 "upstream_error",
1316 )
1317 }
1318 LlmClientError::ResponseTranslation(message) => error_response(
1319 StatusCode::BAD_GATEWAY,
1320 message,
1321 "upstream_error",
1322 "upstream_error",
1323 ),
1324 LlmClientError::Timeout { source } => error_response(
1325 StatusCode::GATEWAY_TIMEOUT,
1326 source.to_string(),
1327 "upstream_error",
1328 "upstream_timeout",
1329 ),
1330 LlmClientError::RequestEncoding(message) => server_error(message),
1331 _ => server_error(error.to_string()),
1332 }
1333}
1334
1335fn upstream_error(status: StatusCode, body: &str) -> Response {
1337 let parsed = serde_json::from_str::<Value>(body).unwrap_or_default();
1338 let error = &parsed["error"];
1339 let message = error["message"].as_str().unwrap_or(body);
1340 let code = error["code"]
1341 .as_str()
1342 .filter(|code| !code.is_empty())
1343 .unwrap_or("upstream_error");
1344 error_response(status, message, "upstream_error", code)
1345}
1346
1347#[derive(Clone)]
1349struct ApiError {
1350 status: StatusCode,
1351 message: String,
1352 error_type: &'static str,
1353 code: String,
1354}
1355
1356impl ApiError {
1357 fn new(
1358 status: StatusCode,
1359 message: impl Into<String>,
1360 error_type: &'static str,
1361 code: impl Into<String>,
1362 ) -> Self {
1363 Self {
1364 status,
1365 message: message.into(),
1366 error_type,
1367 code: code.into(),
1368 }
1369 }
1370
1371 fn into_response(self, wire_format: WireFormat) -> Response {
1372 let body = match wire_format {
1373 WireFormat::AnthropicMessages => json!({
1374 "type": "error",
1375 "error": {
1376 "type": anthropic_error_type(self.status),
1377 "message": self.message.clone(),
1378 }
1379 }),
1380 WireFormat::OpenAiChat | WireFormat::OpenAiResponses => json!({
1381 "error": {
1382 "message": self.message.clone(),
1383 "type": self.error_type,
1384 "code": self.code,
1385 }
1386 }),
1387 };
1388 let mut response = (self.status, Json(body)).into_response();
1389 response
1390 .extensions_mut()
1391 .insert(RequestLogError(self.message.clone()));
1392 response.extensions_mut().insert(self);
1393 response
1394 }
1395}
1396
1397fn render_error_response(response: Response, wire_format: WireFormat) -> Response {
1398 let Some(error) = response.extensions().get::<ApiError>().cloned() else {
1399 return response;
1400 };
1401 error.into_response(wire_format)
1402}
1403
1404fn anthropic_error_response(response: Response) -> Response {
1405 render_error_response(response, WireFormat::AnthropicMessages)
1406}
1407
1408fn anthropic_error_type(status: StatusCode) -> &'static str {
1409 match status {
1410 StatusCode::BAD_REQUEST => "invalid_request_error",
1411 StatusCode::UNAUTHORIZED => "authentication_error",
1412 StatusCode::FORBIDDEN => "permission_error",
1413 StatusCode::NOT_FOUND => "not_found_error",
1414 StatusCode::PAYLOAD_TOO_LARGE => "request_too_large",
1415 StatusCode::TOO_MANY_REQUESTS => "rate_limit_error",
1416 status if status.as_u16() == 529 => "overloaded_error",
1417 _ => "api_error",
1418 }
1419}
1420
1421fn server_error(message: impl Into<String>) -> Response {
1422 error_response(
1423 StatusCode::INTERNAL_SERVER_ERROR,
1424 message,
1425 "server_error",
1426 "server_error",
1427 )
1428}
1429
1430fn invalid_body_error(status: StatusCode, message: impl Into<String>) -> Response {
1431 error_response(status, message, "invalid_request_error", "invalid_body")
1432}
1433
1434fn error_response(
1435 status: StatusCode,
1436 message: impl Into<String>,
1437 error_type: &'static str,
1438 code: impl Into<String>,
1439) -> Response {
1440 ApiError::new(status, message, error_type, code).into_response(WireFormat::OpenAiChat)
1441}
1442
1443async fn models(State(state): State<ServerState>) -> Json<Value> {
1444 Json(model_list_payload(
1445 state
1446 .runner
1447 .models()
1448 .map(|model| (model.id.as_str(), model.capabilities)),
1449 ))
1450}
1451
1452async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {
1453 Json(state.stats.snapshot())
1454}
1455
1456async fn reset_stats(State(state): State<ServerState>) -> Json<Value> {
1457 state.stats.reset();
1458 Json(json!({"status": "reset"}))
1459}
1460
1461#[derive(Deserialize)]
1462struct SessionStatsQuery {
1463 session_id: String,
1464}
1465
1466async fn get_session_stats(
1469 State(state): State<ServerState>,
1470 query: std::result::Result<Query<SessionStatsQuery>, QueryRejection>,
1471) -> Response {
1472 let Query(query) = match query {
1473 Ok(query) => query,
1474 Err(error) => {
1475 return error_response(
1476 StatusCode::BAD_REQUEST,
1477 error.to_string(),
1478 "invalid_request_error",
1479 "invalid_query",
1480 );
1481 }
1482 };
1483 let Some(routing_log) = state.routing_log else {
1484 return not_found().await;
1486 };
1487 let session_id = query.session_id.clone();
1488 let snapshot =
1490 match task::spawn_blocking(move || routing_log.snapshot_session(&session_id)).await {
1491 Ok(s) => s,
1492 Err(err) => {
1493 return server_error(format!("failed to snapshot: {err}"));
1494 }
1495 };
1496
1497 match snapshot {
1498 Ok(Some(snapshot)) => (StatusCode::OK, Json(snapshot)).into_response(),
1499 Ok(None) => error_response(
1500 StatusCode::NOT_FOUND,
1501 "Routing session not found",
1502 "not_found",
1503 "routing_session_not_found",
1504 ),
1505 Err(error) => server_error(format!("failed to read routing log: {error}")),
1506 }
1507}
1508
1509async fn health() -> Json<Value> {
1510 Json(json!({"status": "ok"}))
1511}
1512
1513async fn prometheus_metrics(State(state): State<ServerState>) -> Response {
1514 match metrics::encode(&state.metrics) {
1515 Ok(body) => ([(CONTENT_TYPE, metrics::CONTENT_TYPE)], body).into_response(),
1516 Err(error) => server_error(error),
1517 }
1518}
1519
1520async fn not_found() -> Response {
1521 error_response(
1522 StatusCode::NOT_FOUND,
1523 "Not Found",
1524 "not_found",
1525 "endpoint_not_found",
1526 )
1527}
1528
1529fn model_list_payload<'a>(
1530 entries: impl IntoIterator<Item = (&'a str, ModelCapabilities)>,
1531) -> Value {
1532 let mut entries = entries.into_iter().collect::<Vec<_>>();
1533 entries.sort_unstable_by_key(|(model_id, _)| *model_id);
1534 let model_ids = entries.iter().map(|(model, _)| *model).collect::<Vec<_>>();
1535 let first_id = model_ids.first().copied();
1536 let last_id = model_ids.last().copied();
1537 json!({
1538 "object": "list",
1539 "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::<Vec<_>>(),
1540 "models": [],
1542 "first_id": first_id,
1543 "last_id": last_id,
1544 "has_more": false,
1545 "default_model": first_id,
1546 "model_pool": model_ids,
1547 })
1548}
1549
1550fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {
1551 json!({
1552 "id": model,
1553 "object": "model",
1554 "type": "model",
1555 "created": 0,
1556 "owned_by": "switchyard",
1557 "display_name": model,
1558 "capabilities": {
1559 "streaming": true,
1560 "tool_calling": capabilities.tool_calling,
1561 "vision": capabilities.vision,
1562 "context_window": capabilities.context_window,
1563 "supported_inbound_formats": [
1564 "openai-chat-completions",
1565 "openai-responses",
1566 "anthropic-messages",
1567 ],
1568 },
1569 })
1570}
1571
1572fn startup_banner(options: &ServerRunOptions, state: &ServerState, color: bool) -> String {
1573 let scheme = if options.is_tls() { "https" } else { "http" };
1574 let listen_url = url_for_addr(scheme, options.addr);
1575 let request_url = request_url_for_addr(scheme, options.addr);
1576 let routes = state.models().collect::<Vec<_>>();
1577 let route_list = routes.join(", ");
1578 let example_model = routes.first().copied().unwrap_or("switchyard/route");
1579 let example_body = json!({
1580 "model": example_model,
1581 "messages": [{"role": "user", "content": "Hello from Switchyard"}],
1582 });
1583 let example_url = shell_quote(&format!("{request_url}/v1/chat/completions"));
1584 let example_body = shell_quote(&example_body.to_string());
1585 format!(
1586 "{}\nSwitchyard libsy server\n listening: {}\n routes: {}\n\nendpoints:\n{}\n\nexample:\n curl -s {} \\\n -H 'Content-Type: application/json' \\\n -d {}",
1587 render_startup_banner_art(color),
1588 listen_url,
1589 route_list,
1590 endpoint_listing(state.routing_log.is_some()),
1591 example_url,
1592 example_body,
1593 )
1594}
1595
1596fn render_startup_banner_art(color: bool) -> String {
1598 let banner = STARTUP_BANNER_ART.trim_end();
1599 if !color {
1600 return banner.to_string();
1601 }
1602
1603 let (red, green, blue) = (118, 185, 0);
1604 let mut rendered = String::new();
1605 for line in banner.lines() {
1606 rendered.push_str(&format!("\x1b[38;2;{red};{green};{blue}m{line}\x1b[0m\n"));
1607 }
1608 rendered.trim_end_matches('\n').to_string()
1609}
1610
1611fn dry_run_summary(state: &ServerState) -> String {
1612 format!(
1613 "server OK: {}",
1614 state.models().collect::<Vec<_>>().join(", ")
1615 )
1616}
1617
1618fn url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
1619 format!("{scheme}://{}:{}", host_for_url(addr.ip()), addr.port())
1620}
1621
1622fn request_url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
1624 let host = if addr.ip().is_unspecified() {
1625 match addr.ip() {
1626 std::net::IpAddr::V4(_) => "127.0.0.1".to_string(),
1627 std::net::IpAddr::V6(_) => "[::1]".to_string(),
1628 }
1629 } else {
1630 host_for_url(addr.ip())
1631 };
1632 format!("{scheme}://{host}:{}", addr.port())
1633}
1634
1635fn host_for_url(ip: std::net::IpAddr) -> String {
1637 match ip {
1638 std::net::IpAddr::V4(ip) => ip.to_string(),
1639 std::net::IpAddr::V6(ip) => format!("[{ip}]"),
1640 }
1641}
1642
1643fn shell_quote(value: &str) -> String {
1644 format!("'{}'", value.replace('\'', "'\\''"))
1645}
1646
1647fn endpoint_listing(has_routing_log: bool) -> String {
1648 let mut endpoints = vec![
1649 " POST /v1/chat/completions OpenAI Chat Completions",
1650 " POST /v1/messages Anthropic Messages",
1651 " POST /v1/responses OpenAI Responses",
1652 " POST /v1/messages/count_tokens",
1653 " POST /v1/responses/input_tokens",
1654 " POST /v1/responses/compact",
1655 " ANY unmatched paths optional fallback client",
1656 " GET /v1/models configured routes",
1657 " GET /v1/stats routing stats",
1658 " POST /v1/stats/reset",
1659 " GET /metrics Prometheus metrics",
1660 " GET /health",
1661 ];
1662 if has_routing_log {
1663 endpoints.push(" GET /v1/routing/session-stats");
1664 }
1665 endpoints.join("\n")
1666}
1667
1668#[cfg(test)]
1669mod tests {
1670 use switchyard_llm_client::LlmCallObservation;
1671 use tokio::io::{AsyncReadExt, AsyncWriteExt};
1672 use tokio::sync::{Notify, oneshot};
1673
1674 use super::*;
1675
1676 #[test]
1681 fn stats_observer_logs_judge_calls_to_the_routing_log() {
1682 let dir = tempfile::tempdir().expect("temp dir");
1683 let log = SharedRoutingLog::new(dir.path().join("routing.jsonl")).expect("routing log");
1684 let mut headers = HeaderMap::new();
1685 headers.insert("proxy_x_session_id", "session-1".parse().expect("header"));
1686 let metadata = metadata_from_headers(headers);
1687 let context = routing_log::RoutingLogContext::from_metadata(&metadata);
1688 let observer = stats_observer(StatsAccumulator::default(), Some((log.clone(), context)));
1689
1690 let call = |model: &str, answer: bool| {
1691 let observation = LlmCallObservation {
1692 selected_model: ModelId::from(model),
1693 is_success: true,
1694 duration: Duration::from_millis(3),
1695 usage: Some(Usage {
1696 input_tokens: Some(100),
1697 output_tokens: Some(7),
1698 ..Usage::default()
1699 }),
1700 };
1701 if answer {
1702 RunObservation::AnswerCall(observation)
1703 } else {
1704 RunObservation::LlmCall(observation)
1705 }
1706 };
1707 observer(call("judge-model", false));
1708 observer(call("routed-model", true));
1709
1710 let snapshot = log
1711 .snapshot_session("session-1")
1712 .expect("read log")
1713 .expect("session recorded");
1714 let snapshot = serde_json::to_value(&snapshot).expect("serializable snapshot");
1715 assert_eq!(snapshot["models"]["judge-model"]["calls"], 1);
1716 assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100);
1717 assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7);
1718 assert!(snapshot["models"].get("routed-model").is_none());
1719 }
1720
1721 #[derive(Clone)]
1722 struct ShutdownTestState {
1723 started: Arc<Notify>,
1724 release: Arc<Notify>,
1725 }
1726
1727 struct ShutdownTestServer {
1728 state: ShutdownTestState,
1729 shutdown: oneshot::Sender<()>,
1730 server: task::JoinHandle<ServerResult<()>>,
1731 request: task::JoinHandle<std::io::Result<Vec<u8>>>,
1732 }
1733
1734 async fn blocked_request(State(state): State<ShutdownTestState>) -> &'static str {
1735 state.started.notify_one();
1736 state.release.notified().await;
1737 "done"
1738 }
1739
1740 async fn raw_request(addr: SocketAddr) -> std::io::Result<Vec<u8>> {
1741 let mut stream = tokio::net::TcpStream::connect(addr).await?;
1742 stream
1743 .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
1744 .await?;
1745 let mut response = Vec::new();
1746 stream.read_to_end(&mut response).await?;
1747 Ok(response)
1748 }
1749
1750 fn shutdown_test_server(shutdown_timeout: Duration) -> ShutdownTestServer {
1751 let state = ShutdownTestState {
1752 started: Arc::new(Notify::new()),
1753 release: Arc::new(Notify::new()),
1754 };
1755 let router = Router::new()
1756 .route("/", get(blocked_request))
1757 .with_state(state.clone());
1758 let listener = bind_tcp_listener("127.0.0.1:0".parse().expect("valid address"), 16)
1759 .expect("listener binds");
1760 let addr = listener.local_addr().expect("listener has an address");
1761 let (shutdown, shutdown_receiver) = oneshot::channel();
1762 let server = tokio::spawn(serve(listener, router, shutdown_timeout, async move {
1763 let _ = shutdown_receiver.await;
1764 }));
1765 let request = tokio::spawn(raw_request(addr));
1766 ShutdownTestServer {
1767 state,
1768 shutdown,
1769 server,
1770 request,
1771 }
1772 }
1773
1774 #[tokio::test]
1776 async fn shutdown_drains_until_configured_deadline() {
1777 let ShutdownTestServer {
1778 state,
1779 shutdown,
1780 mut server,
1781 request,
1782 } = shutdown_test_server(Duration::from_secs(1));
1783 state.started.notified().await;
1784 shutdown.send(()).expect("server receives shutdown");
1785 assert!(
1786 tokio::time::timeout(Duration::from_millis(25), &mut server)
1787 .await
1788 .is_err(),
1789 "server must wait for the active request"
1790 );
1791 state.release.notify_one();
1792 tokio::time::timeout(Duration::from_secs(1), server)
1793 .await
1794 .expect("server stops after request drains")
1795 .expect("server task completes")
1796 .expect("server exits cleanly");
1797 let response = request
1798 .await
1799 .expect("request task completes")
1800 .expect("request succeeds");
1801 assert!(response.windows(8).any(|part| part == b"200 OK\r\n"));
1802 assert!(response.ends_with(b"done"));
1803
1804 let ShutdownTestServer {
1805 state,
1806 shutdown,
1807 server,
1808 request,
1809 } = shutdown_test_server(Duration::from_millis(25));
1810 state.started.notified().await;
1811 shutdown.send(()).expect("server receives shutdown");
1812 tokio::time::timeout(Duration::from_secs(1), server)
1813 .await
1814 .expect("shutdown deadline is enforced")
1815 .expect("server task completes")
1816 .expect("server exits cleanly");
1817 state.release.notify_one();
1818 request.abort();
1819 }
1820
1821 #[derive(Clone)]
1822 struct CancelTestState {
1823 started: Arc<Notify>,
1824 cancelled: Arc<Notify>,
1825 }
1826
1827 struct CancelSentinel(Arc<Notify>);
1830
1831 impl Drop for CancelSentinel {
1832 fn drop(&mut self) {
1833 self.0.notify_one();
1834 }
1835 }
1836
1837 async fn never_responds(State(state): State<CancelTestState>) -> &'static str {
1838 let _sentinel = CancelSentinel(state.cancelled.clone());
1839 state.started.notify_one();
1840 std::future::pending::<()>().await;
1841 "unreachable"
1842 }
1843
1844 #[tokio::test]
1848 async fn client_disconnect_cancels_a_buffered_handler() {
1849 let state = CancelTestState {
1850 started: Arc::new(Notify::new()),
1851 cancelled: Arc::new(Notify::new()),
1852 };
1853 let router = Router::new()
1854 .route("/", post(never_responds))
1855 .with_state(state.clone());
1856 let listener = bind_tcp_listener("127.0.0.1:0".parse().expect("valid address"), 16)
1857 .expect("listener binds");
1858 let addr = listener.local_addr().expect("listener has an address");
1859 let (shutdown, shutdown_receiver) = oneshot::channel();
1860 let server = tokio::spawn(serve(
1861 listener,
1862 router,
1863 Duration::from_millis(25),
1864 async move {
1865 let _ = shutdown_receiver.await;
1866 },
1867 ));
1868
1869 let mut stream = tokio::net::TcpStream::connect(addr)
1870 .await
1871 .expect("client connects");
1872 stream
1873 .write_all(b"POST / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}")
1874 .await
1875 .expect("client sends a complete request");
1876 state.started.notified().await;
1877
1878 drop(stream);
1880
1881 tokio::time::timeout(Duration::from_secs(5), state.cancelled.notified())
1882 .await
1883 .expect("handler future is dropped when the client disconnects");
1884
1885 let _ = shutdown.send(());
1886 let _ = tokio::time::timeout(Duration::from_secs(5), server).await;
1887 }
1888
1889 #[derive(Clone, Default)]
1891 struct CapturedEvents(Arc<Mutex<Vec<(Level, String)>>>);
1892
1893 impl tracing_subscriber::Layer<tracing_subscriber::Registry> for CapturedEvents {
1894 fn on_event(
1895 &self,
1896 event: &tracing::Event<'_>,
1897 _ctx: tracing_subscriber::layer::Context<'_, tracing_subscriber::Registry>,
1898 ) {
1899 if event.metadata().target() != "switchyard_server::request" {
1900 return;
1901 }
1902 let mut message = String::new();
1903 event.record(
1904 &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
1905 if field.name() == "message" {
1906 message = format!("{value:?}");
1907 }
1908 },
1909 );
1910 self.0.lock().push((*event.metadata().level(), message));
1911 }
1912 }
1913
1914 fn request_log_context() -> RequestLogContext {
1915 RequestLogContext {
1916 started: Instant::now(),
1917 wire_format: WireFormat::OpenAiChat,
1918 requested_model: Some("switchyard/route".to_string()),
1919 streaming: false,
1920 session_id: Some("session".to_string()),
1921 correlation_id: None,
1922 }
1923 }
1924
1925 fn captured_events(run: impl FnOnce()) -> Vec<(Level, String)> {
1926 use tracing_subscriber::layer::SubscriberExt;
1927
1928 let captured = CapturedEvents::default();
1929 let subscriber = tracing_subscriber::registry().with(captured.clone());
1930 tracing::subscriber::with_default(subscriber, run);
1931 captured.0.lock().clone()
1932 }
1933
1934 #[test]
1937 fn dropping_the_request_log_guard_reports_a_cancelled_request() {
1938 let events = captured_events(|| {
1939 drop(RequestLogGuard(Some(request_log_context())));
1940 });
1941
1942 assert_eq!(events.len(), 1);
1943 assert_eq!(events[0].0, Level::WARN);
1944 assert!(events[0].1.contains("cancelled"), "{:?}", events[0].1);
1945 }
1946
1947 #[test]
1950 fn an_answered_request_reports_only_its_response() {
1951 let events = captured_events(|| {
1952 let mut guard = RequestLogGuard(Some(request_log_context()));
1953 guard.emit(&StatusCode::OK.into_response());
1954 });
1955
1956 assert_eq!(events.len(), 1);
1957 assert_eq!(events[0].0, Level::INFO);
1958 assert!(events[0].1.contains("handled"), "{:?}", events[0].1);
1959 }
1960
1961 #[test]
1963 fn request_log_level_follows_http_status() {
1964 assert_eq!(request_log_level(StatusCode::OK), Level::INFO);
1965 assert_eq!(request_log_level(StatusCode::BAD_REQUEST), Level::WARN);
1966 assert_eq!(
1967 request_log_level(StatusCode::INTERNAL_SERVER_ERROR),
1968 Level::ERROR
1969 );
1970 }
1971
1972 #[test]
1974 fn error_response_carries_request_log_error() {
1975 let response = error_response(
1976 StatusCode::BAD_REQUEST,
1977 "invalid request",
1978 "invalid_request_error",
1979 "invalid_request_error",
1980 );
1981
1982 assert_eq!(
1983 response
1984 .extensions()
1985 .get::<RequestLogError>()
1986 .map(|error| error.0.as_str()),
1987 Some("invalid request")
1988 );
1989 }
1990}