Skip to main content

switchyard_server/
lib.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Rust HTTP server for libsy algorithms.
5
6mod capabilities;
7pub mod config;
8mod metrics;
9mod observability;
10mod redaction;
11mod response;
12mod routing_log;
13mod shutdown;
14mod sse;
15mod stats;
16mod usage_metrics;
17
18use std::collections::BTreeMap;
19use std::error::Error;
20use std::fmt::{Display, Formatter};
21use std::future::Future;
22use std::io::IsTerminal;
23use std::net::SocketAddr;
24use std::path::PathBuf;
25use std::sync::Arc;
26use std::time::{Duration, Instant};
27
28use axum::body::Body;
29use axum::extract::rejection::{JsonRejection, QueryRejection};
30use axum::extract::{DefaultBodyLimit, Query, Request as HttpRequest, State};
31use axum::http::header::CONTENT_TYPE;
32use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
33use axum::middleware::Next;
34use axum::response::{IntoResponse, Response};
35use axum::routing::{get, post};
36use axum::{Extension, Json, Router};
37use axum_server::tls_rustls::RustlsConfig;
38use libsy::{LibsyError, RoutingOutcome};
39use parking_lot::Mutex;
40use serde::{Deserialize, Serialize};
41use serde_json::{Value, json};
42use switchyard_llm_client::{AuxiliaryOperation, RunObservation, RunObserver};
43use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};
44use switchyard_runner::{
45    CallerAuthKind, DecisionTarget, ModelCapabilities, Route, RunOutput, Runner, RunnerError,
46};
47use tokio::net::{TcpListener, TcpSocket};
48use tokio::task;
49use tracing::{Instrument, Level};
50
51use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response};
52
53use crate::response::into_http_response;
54use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env};
55
56pub use observability::{flush_observability, initialize_observability};
57
58/// Default TCP listen backlog used by the Rust server.
59pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;
60
61/// Default time allowed for active requests to finish during shutdown.
62pub const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
63
64/// Maximum buffered JSON request size accepted by the LLM endpoints.
65pub const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024;
66
67const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model";
68const FORWARDED_UPSTREAM_HEADERS: &[&str] = &[
69    "baggage",
70    "openai-processing-ms",
71    // Anthropic spells its correlation id without the `x-` prefix.
72    "request-id",
73    "traceparent",
74    "tracestate",
75    "x-request-id",
76];
77const FORWARDED_UPSTREAM_HEADER_PREFIXES: &[&str] =
78    &["anthropic-ratelimit-", "x-ratelimit-", "x-upstream-"];
79const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512;
80
81/// Whether an upstream header is safe and useful to expose downstream.
82fn should_forward_upstream_header(name: &HeaderName) -> bool {
83    let name = name.as_str();
84    FORWARDED_UPSTREAM_HEADERS.contains(&name)
85        || FORWARDED_UPSTREAM_HEADER_PREFIXES
86            .iter()
87            .any(|prefix| name.starts_with(prefix))
88}
89/// Non-standard status used only in logs and metrics for a request whose
90/// downstream client disconnected before any response was written.
91const CLIENT_CLOSED_REQUEST: u16 = 499;
92const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt");
93
94/// Error returned while configuring or running the server.
95#[derive(Debug)]
96pub struct ServerError {
97    message: String,
98}
99
100impl ServerError {
101    /// Creates a server error with a user-facing message.
102    pub fn new(message: impl Into<String>) -> Self {
103        Self {
104            message: message.into(),
105        }
106    }
107}
108
109impl Display for ServerError {
110    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
111        formatter.write_str(&self.message)
112    }
113}
114
115impl Error for ServerError {}
116
117impl From<RunnerError> for ServerError {
118    fn from(error: RunnerError) -> Self {
119        let mut message = error.to_string();
120        let mut source = error.source();
121        while let Some(error) = source {
122            message.push_str(": ");
123            message.push_str(&error.to_string());
124            source = error.source();
125        }
126        Self::new(message)
127    }
128}
129
130/// Result returned by server setup and lifecycle operations.
131pub type ServerResult<T> = std::result::Result<T, ServerError>;
132
133/// Decision result using the same selected/fallback order as libsy.
134#[derive(Serialize)]
135struct DecisionResponse {
136    selected: DecisionTargetResponse,
137    fallbacks: Vec<DecisionTargetResponse>,
138    #[serde(skip_serializing_if = "Option::is_none")]
139    response: Option<Value>,
140}
141
142/// Response view over one target's existing configuration.
143#[derive(Serialize)]
144struct DecisionTargetResponse {
145    target: String,
146    model: ModelId,
147    llm_client: DecisionLlmClientResponse,
148    extra_body: BTreeMap<String, Value>,
149}
150
151/// Non-secret client settings needed to call a selected model.
152#[derive(Serialize)]
153struct DecisionLlmClientResponse {
154    format: WireFormat,
155    base_url: String,
156}
157
158/// Shared server state used by all endpoint handlers.
159#[derive(Clone)]
160pub struct ServerState {
161    runner: Arc<Runner>,
162    redactor: Arc<redaction::Redactor>,
163    fallback_http: reqwest::Client,
164    metrics: prometheus::Registry,
165    stats: StatsAccumulator,
166    routing_log: Option<SharedRoutingLog>,
167    track_cache_eligibility: bool,
168}
169
170#[derive(Clone)]
171struct SharedRoutingLog {
172    writer: Arc<Mutex<routing_log::RoutingLog>>,
173    path: PathBuf,
174}
175
176impl SharedRoutingLog {
177    fn new(path: PathBuf) -> ServerResult<Self> {
178        Ok(Self {
179            writer: Arc::new(Mutex::new(routing_log::RoutingLog::new(path.clone())?)),
180            path,
181        })
182    }
183
184    fn append(
185        &self,
186        context: routing_log::RoutingLogContext,
187        model: &str,
188        tier: Option<&str>,
189        usage: &Usage,
190    ) {
191        if let Err(error) = self.writer.lock().append(context, model, tier, usage) {
192            tracing::warn!(path = %self.path.display(), %error, "routing log append failed");
193        }
194    }
195
196    fn snapshot_session(
197        &self,
198        session_id: &str,
199    ) -> std::io::Result<Option<routing_log::SessionStatsSnapshot>> {
200        routing_log::snapshot(&self.path, session_id)
201    }
202}
203
204impl ServerState {
205    /// Creates HTTP-server state around an already configured runner.
206    pub fn from_runner(runner: Runner) -> ServerResult<Self> {
207        let metrics = metrics::registry().map_err(ServerError::new)?;
208        let fallback_http = reqwest::Client::builder()
209            .redirect(reqwest::redirect::Policy::none())
210            .build()
211            .map_err(|error| ServerError::new(error.to_string()))?;
212        let stats = StatsAccumulator::new(
213            metrics.clone(),
214            runner.models().map(|model| model.algorithm),
215        );
216        let redactor = redaction::Redactor::new(runner.provider_api_keys());
217        Ok(Self {
218            redactor: Arc::new(redactor),
219            runner: Arc::new(runner),
220            fallback_http,
221            metrics,
222            stats,
223            routing_log: None,
224            track_cache_eligibility: tracking_enabled_from_env(),
225        })
226    }
227
228    /// Enables durable per-request routing records at `path`.
229    pub fn with_routing_log(mut self, path: impl Into<PathBuf>) -> ServerResult<Self> {
230        self.routing_log = Some(SharedRoutingLog::new(path.into())?);
231        Ok(self)
232    }
233
234    /// Returns the route model IDs served by the configured algorithms.
235    pub fn models(&self) -> impl Iterator<Item = &str> {
236        self.runner.models().map(|model| model.id.as_str())
237    }
238
239    /// Returns the caller credential family used by `model`, if any.
240    pub fn caller_auth_kind(&self, model: &str) -> Option<&'static str> {
241        self.runner
242            .route(model)
243            .and_then(|route| route.caller_auth())
244            .map(|kind| kind.as_str())
245    }
246
247    fn route_for_model(&self, model: &str) -> Option<&Route> {
248        self.runner.route(model)
249    }
250
251    fn decision_response(
252        &self,
253        route_model: &ModelId,
254        outcome: &RoutingOutcome,
255        response: Option<Value>,
256    ) -> Option<DecisionResponse> {
257        let description = self.runner.describe_decision(route_model, outcome)?;
258        let convert = |target: DecisionTarget| DecisionTargetResponse {
259            target: target.target,
260            model: target.model,
261            llm_client: DecisionLlmClientResponse {
262                format: target.format,
263                base_url: target.base_url,
264            },
265            extra_body: target.extra_body,
266        };
267        Some(DecisionResponse {
268            selected: convert(description.selected),
269            fallbacks: description.fallbacks.into_iter().map(convert).collect(),
270            response,
271        })
272    }
273}
274/// Runtime options shared by server entry points.
275#[derive(Clone, Debug)]
276pub struct ServerRunOptions {
277    /// Socket address to bind.
278    pub addr: SocketAddr,
279    /// TCP listen backlog.
280    pub backlog: u32,
281    /// Validate runtime construction without binding a socket.
282    pub dry_run: bool,
283    /// Maximum time active requests may drain after shutdown begins.
284    pub shutdown_timeout: Duration,
285    /// TLS certificate configuration, when HTTPS is enabled.
286    pub tls: Option<TlsOptions>,
287}
288
289/// TLS certificate paths used by the server.
290#[derive(Clone, Debug)]
291pub struct TlsOptions {
292    /// TLS certificate path in PEM format.
293    pub cert: PathBuf,
294    /// TLS private-key path in PEM format.
295    pub key: PathBuf,
296}
297
298impl ServerRunOptions {
299    fn is_tls(&self) -> bool {
300        self.tls.is_some()
301    }
302}
303
304/// Validates the runtime and starts the HTTP server unless `dry_run` is set.
305pub async fn run_server(state: ServerState, options: ServerRunOptions) -> ServerResult<()> {
306    if options.dry_run {
307        println!("{}", dry_run_summary(&state));
308        return Ok(());
309    }
310
311    let server = BoundServer::bind(state, options)?;
312    println!("{}", server.startup_banner(std::io::stdout().is_terminal()));
313    server.serve(shutdown::signal()).await
314}
315
316/// A configured server with its listening socket already bound.
317pub struct BoundServer {
318    listener: TcpListener,
319    router: Router,
320    options: ServerRunOptions,
321    state: ServerState,
322}
323
324impl BoundServer {
325    /// Binds the configured address and prepares the HTTP router.
326    pub fn bind(state: ServerState, options: ServerRunOptions) -> ServerResult<Self> {
327        let listener = bind_tcp_listener(options.addr, options.backlog)?;
328        let addr = listener.local_addr().map_err(server_io_error)?;
329        Ok(Self {
330            listener,
331            router: build_switchyard_router(state.clone()),
332            options: ServerRunOptions { addr, ..options },
333            state,
334        })
335    }
336
337    /// Returns the actual bound address, including an OS-selected port.
338    pub fn local_addr(&self) -> SocketAddr {
339        self.options.addr
340    }
341
342    /// Serves requests until the supplied shutdown future resolves.
343    pub async fn serve(
344        self,
345        shutdown: impl Future<Output = ()> + Send + 'static,
346    ) -> ServerResult<()> {
347        let shutdown_timeout = self.options.shutdown_timeout;
348        if let Some(tls) = self.options.tls {
349            serve_tls(self.listener, self.router, tls, shutdown_timeout, shutdown).await
350        } else {
351            serve(self.listener, self.router, shutdown_timeout, shutdown).await
352        }
353    }
354
355    fn startup_banner(&self, color: bool) -> String {
356        startup_banner(&self.options, &self.state, color)
357    }
358}
359
360async fn serve_tls(
361    listener: TcpListener,
362    router: Router,
363    tls: TlsOptions,
364    shutdown_timeout: Duration,
365    shutdown: impl Future<Output = ()> + Send + 'static,
366) -> ServerResult<()> {
367    if let Err(error) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
368        tracing::debug!(?error, "TLS crypto provider was already installed");
369    }
370
371    let config = RustlsConfig::from_pem_file(tls.cert, tls.key)
372        .await
373        .map_err(server_io_error)?;
374    let std_listener = listener.into_std().map_err(server_io_error)?;
375    let server = axum_server::from_tcp_rustls(std_listener, config).map_err(server_io_error)?;
376    let handle = axum_server::Handle::new();
377    let server = server
378        .handle(handle.clone())
379        .serve(router.into_make_service());
380    serve_until_shutdown(server, handle, shutdown_timeout, shutdown).await
381}
382
383async fn serve(
384    listener: TcpListener,
385    router: Router,
386    shutdown_timeout: Duration,
387    shutdown: impl Future<Output = ()> + Send + 'static,
388) -> ServerResult<()> {
389    let std_listener = listener.into_std().map_err(server_io_error)?;
390    let server = axum_server::from_tcp(std_listener).map_err(server_io_error)?;
391    let handle = axum_server::Handle::new();
392    let server = server
393        .handle(handle.clone())
394        .serve(router.into_make_service());
395    serve_until_shutdown(server, handle, shutdown_timeout, shutdown).await
396}
397
398/// Runs the server until it exits or shutdown begins, then drains active requests.
399async fn serve_until_shutdown(
400    server: impl Future<Output = std::io::Result<()>>,
401    handle: axum_server::Handle<SocketAddr>,
402    timeout: Duration,
403    shutdown: impl Future<Output = ()> + Send + 'static,
404) -> ServerResult<()> {
405    tokio::pin!(server);
406    tokio::select! {
407        result = &mut server => result.map_err(server_io_error),
408        _ = shutdown => {
409            tracing::info!(
410                ?timeout,
411                "shutdown signal received; draining active requests"
412            );
413            handle.graceful_shutdown(Some(timeout));
414            server.await.map_err(server_io_error)
415        }
416    }
417}
418
419/// Ingress timestamp for one request, taken before any body is read.
420#[derive(Clone, Copy)]
421struct RequestStart(Instant);
422
423/// Routing-log marker distinguishing classifier and judge calls from answer calls.
424const CLASSIFIER_TIER: &str = "classifier";
425
426/// Maps answer-call observations to backend stats, routing calls to classifier/judge
427/// stats, and records routing overhead once the algorithm run completes.
428///
429/// Successful classifier/judge calls are also appended to `classifier_log` when one is
430/// configured, so per-session routing snapshots account for judge token overhead. Routed
431/// calls stay off this path: the served call is logged with its terminal usage in
432/// [`usage_metrics::observe`], which would make a second append here a double count.
433fn stats_observer(
434    stats: StatsAccumulator,
435    classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>,
436) -> RunObserver {
437    Arc::new(move |observation| match observation {
438        RunObservation::Outcome(_) => {}
439        RunObservation::AnswerCall(call) => {
440            let latency_ms = call.duration.as_secs_f64() * 1_000.0;
441            if call.is_success {
442                stats.record_success(&call.selected_model, latency_ms);
443            } else {
444                stats.record_error(&call.selected_model);
445            }
446        }
447        RunObservation::LlmCall(call) => {
448            let latency_ms = call.duration.as_secs_f64() * 1_000.0;
449            if call.is_success {
450                if let (Some((log, context)), Some(usage)) =
451                    (classifier_log.as_ref(), call.usage.as_ref())
452                {
453                    log.append(
454                        context.clone(),
455                        &call.selected_model,
456                        Some(CLASSIFIER_TIER),
457                        usage,
458                    );
459                }
460                stats.record_classifier_success(
461                    call.selected_model,
462                    call.usage.as_ref().map(usage_metrics::token_usage),
463                    latency_ms,
464                );
465            } else {
466                stats.record_classifier_error(call.selected_model);
467            }
468        }
469        RunObservation::RoutingOverhead(duration) => {
470            stats.record_routing_overhead(duration.as_secs_f64() * 1_000.0);
471        }
472    })
473}
474
475/// Stamps the ingress instant into request extensions. Runs as a router layer,
476/// so it executes before the handlers' `Json` extractor buffers the body —
477/// request-latency measurements therefore include body read and decode.
478async fn stamp_request_start(mut request: HttpRequest, next: Next) -> Response {
479    request
480        .extensions_mut()
481        .insert(RequestStart(Instant::now()));
482    next.run(request).await
483}
484
485/// Builds an Axum router containing only the primary LLM inference endpoints.
486///
487/// The registered paths are `/v1/chat/completions`, `/v1/messages`, and `/v1/responses`.
488/// The router excludes operational, discovery, auxiliary, and fallback proxy routes so an
489/// embedding application can mount those capabilities under its own policies.
490pub fn build_llm_router(state: ServerState) -> Router {
491    finish_router(primary_llm_routes(), state)
492}
493
494/// Builds the full Axum router used by the standalone Switchyard server.
495pub fn build_switchyard_router(state: ServerState) -> Router {
496    let mut router = primary_llm_routes()
497        .route("/v1/decision", post(decision))
498        .route("/v1/messages/count_tokens", post(anthropic_count_tokens))
499        .route(
500            "/v1/responses/input_tokens",
501            post(openai_responses_input_tokens),
502        )
503        .route("/v1/responses/compact", post(openai_responses_compact))
504        .route("/v1/models", get(models))
505        .route("/v1/stats", get(get_stats))
506        .route("/v1/stats/reset", post(reset_stats))
507        .route("/metrics", get(prometheus_metrics))
508        .route("/health", get(health));
509    if state.routing_log.is_some() {
510        router = router.route("/v1/routing/session-stats", get(get_session_stats));
511    }
512    finish_router(router.fallback(proxy_unmatched), state)
513}
514
515// Keeps the embedded and standalone servers on the same primary route definitions.
516fn primary_llm_routes() -> Router<ServerState> {
517    Router::new()
518        .route("/v1/chat/completions", post(openai_chat_completions))
519        .route("/v1/messages", post(anthropic_messages))
520        .route("/v1/responses", post(openai_responses))
521}
522
523// Applies the serving limits and request timing shared by both public router constructors.
524fn finish_router(router: Router<ServerState>, state: ServerState) -> Router {
525    router
526        .layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))
527        .layer(axum::middleware::from_fn_with_state(
528            state.clone(),
529            redaction::redact_response,
530        ))
531        // `layer` only wraps routes registered before it, so this stays last.
532        .layer(axum::middleware::from_fn(stamp_request_start))
533        .with_state(state)
534}
535
536fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> ServerResult<TcpListener> {
537    let socket = if addr.is_ipv4() {
538        TcpSocket::new_v4()
539    } else {
540        TcpSocket::new_v6()
541    }
542    .map_err(server_io_error)?;
543
544    socket.set_reuseaddr(true).map_err(server_io_error)?;
545    socket.bind(addr).map_err(server_io_error)?;
546    socket.listen(backlog).map_err(server_io_error)
547}
548
549fn server_io_error(error: std::io::Error) -> ServerError {
550    ServerError::new(error.to_string())
551}
552
553async fn openai_chat_completions(
554    State(state): State<ServerState>,
555    Extension(started): Extension<RequestStart>,
556    headers: HeaderMap,
557    body: std::result::Result<Json<Value>, JsonRejection>,
558) -> Response {
559    handle_endpoint(state, started, headers, body, WireFormat::OpenAiChat).await
560}
561
562async fn anthropic_messages(
563    State(state): State<ServerState>,
564    Extension(started): Extension<RequestStart>,
565    headers: HeaderMap,
566    body: std::result::Result<Json<Value>, JsonRejection>,
567) -> Response {
568    handle_endpoint(state, started, headers, body, WireFormat::AnthropicMessages).await
569}
570
571async fn openai_responses(
572    State(state): State<ServerState>,
573    Extension(started): Extension<RequestStart>,
574    headers: HeaderMap,
575    body: std::result::Result<Json<Value>, JsonRejection>,
576) -> Response {
577    handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await
578}
579
580// Forwards unmatched requests unchanged to the configured API root.
581async fn proxy_unmatched(State(state): State<ServerState>, request: HttpRequest) -> Response {
582    let Some(base_url) = state.runner.fallback_base_url() else {
583        return not_found().await;
584    };
585    let (mut parts, body) = request.into_parts();
586    let path_and_query = parts
587        .uri
588        .path_and_query()
589        .map_or_else(|| parts.uri.path().to_string(), ToString::to_string);
590    strip_hop_by_hop_headers(&mut parts.headers);
591    parts.headers.remove(axum::http::header::HOST);
592    let url = match fallback_url(base_url, &path_and_query) {
593        Ok(url) => url,
594        Err(error) => {
595            return error_response(
596                StatusCode::BAD_GATEWAY,
597                error,
598                "upstream_error",
599                "upstream_error",
600            );
601        }
602    };
603    let body = reqwest::Body::wrap_stream(body.into_data_stream());
604    match state
605        .fallback_http
606        .request(parts.method, url)
607        .headers(parts.headers)
608        .body(body)
609        .send()
610        .await
611    {
612        Ok(response) => {
613            let response: http::Response<reqwest::Body> = response.into();
614            let (mut parts, body) = response.into_parts();
615            strip_hop_by_hop_headers(&mut parts.headers);
616            Response::from_parts(parts, Body::new(body))
617        }
618        Err(error) => error_response(
619            StatusCode::BAD_GATEWAY,
620            error.without_url().to_string(),
621            "upstream_error",
622            "upstream_error",
623        ),
624    }
625}
626
627fn fallback_url(base_url: &str, path_and_query: &str) -> Result<reqwest::Url, String> {
628    let mut url = reqwest::Url::parse(base_url.trim())
629        .map_err(|error| format!("invalid fallback base URL: {error}"))?;
630    let base_query = url.query().map(str::to_owned);
631    url.set_query(None);
632    url.set_fragment(None);
633
634    let base_path = url.path().trim_end_matches('/');
635    let root_path = [
636        "/v1/chat/completions",
637        "/v1/responses",
638        "/v1/messages",
639        "/v1",
640    ]
641    .iter()
642    .find_map(|suffix| base_path.strip_suffix(suffix))
643    .unwrap_or(base_path);
644    let (request_path, request_query) = path_and_query
645        .split_once('?')
646        .map_or((path_and_query, None), |(path, query)| (path, Some(query)));
647    let request_path = request_path.strip_prefix('/').unwrap_or(request_path);
648    let target_path = if root_path.is_empty() {
649        format!("/{request_path}")
650    } else if request_path.is_empty() {
651        format!("{root_path}/")
652    } else {
653        format!("{root_path}/{request_path}")
654    };
655    url.set_path(&target_path);
656
657    let query = match (base_query, request_query) {
658        (Some(base_query), Some(request_query)) => Some(format!("{base_query}&{request_query}")),
659        (Some(base_query), None) => Some(base_query),
660        (None, Some(request_query)) => Some(request_query.to_owned()),
661        (None, None) => None,
662    };
663    url.set_query(query.as_deref());
664    Ok(url)
665}
666
667fn strip_hop_by_hop_headers(headers: &mut HeaderMap) {
668    let connection_headers = headers
669        .get_all(axum::http::header::CONNECTION)
670        .iter()
671        .filter_map(|value| value.to_str().ok())
672        .flat_map(|value| value.split(','))
673        .filter_map(|name| HeaderName::from_bytes(name.trim().as_bytes()).ok())
674        .collect::<Vec<_>>();
675    for name in connection_headers {
676        headers.remove(name);
677    }
678    for name in [
679        "connection",
680        "keep-alive",
681        "proxy-authenticate",
682        "proxy-authorization",
683        "proxy-connection",
684        "te",
685        "trailer",
686        "transfer-encoding",
687        "upgrade",
688    ] {
689        headers.remove(name);
690    }
691}
692
693/// One provider request submitted for a routing decision without an answer-model call.
694#[derive(Deserialize)]
695#[serde(deny_unknown_fields)]
696struct DecisionEndpointRequest {
697    input_format: WireFormat,
698    request: Value,
699}
700
701/// Selects a target while still allowing the algorithm's classifier and judge calls.
702async fn decision(
703    State(state): State<ServerState>,
704    headers: HeaderMap,
705    body: std::result::Result<Json<DecisionEndpointRequest>, JsonRejection>,
706) -> Response {
707    let body = match body {
708        Ok(Json(body)) if body.request.is_object() => body,
709        Ok(_) => {
710            return invalid_body_error(StatusCode::BAD_REQUEST, "request must be a JSON object");
711        }
712        Err(error) => {
713            return invalid_body_error(
714                error.status(),
715                format!("Request body must be valid JSON: {error}"),
716            );
717        }
718    };
719    let input_format = body.input_format;
720    let (route, request) = match resolve_route(
721        &state,
722        metadata_from_headers(headers),
723        body.request,
724        input_format,
725    ) {
726        Ok(resolved) => resolved,
727        Err(response) => return response,
728    };
729    let route_model = request
730        .llm_request
731        .model
732        .as_deref()
733        .map(ModelId::from)
734        .unwrap_or_default();
735
736    let mut outcome = match route.decide(request).await {
737        Ok(outcome) => outcome,
738        Err(error) => return runner_error(error),
739    };
740    let response = match outcome.response.take() {
741        Some(response) => {
742            let aggregate = match response.llm_response.into_agg().await {
743                Ok(aggregate) => aggregate,
744                Err(error) => return client_error(&error),
745            };
746            // The request moved into the decision run, so its namespace mapping
747            // is gone by here. A Codex tool call in this preview keeps its
748            // qualified name.
749            match encode_aggregated_response(
750                &aggregate,
751                input_format,
752                outcome.selected_model_id().ok().map(ModelId::as_str),
753            ) {
754                Ok(response) => Some(response),
755                Err(error) => return server_error(error.to_string()),
756            }
757        }
758        None => None,
759    };
760    match state.decision_response(&route_model, &outcome, response) {
761        Some(response) => Json(response).into_response(),
762        None => {
763            server_error("routing outcome contains a model with no callable target configuration")
764        }
765    }
766}
767
768/// Anthropic token counting against the route's explicitly configured target.
769async fn anthropic_count_tokens(
770    State(state): State<ServerState>,
771    headers: HeaderMap,
772    body: std::result::Result<Json<Value>, JsonRejection>,
773) -> Response {
774    let body = match llm_json_body(body) {
775        Ok(body) => body,
776        Err((status, message)) => {
777            return anthropic_error_response(invalid_body_error(status, message));
778        }
779    };
780    let (route, request) = match resolve_route(
781        &state,
782        metadata_from_headers(headers),
783        body,
784        WireFormat::AnthropicMessages,
785    ) {
786        Ok(resolved) => resolved,
787        Err(response) => return anthropic_error_response(response),
788    };
789    anthropic_error_response(
790        match route
791            .call_auxiliary(request, AuxiliaryOperation::AnthropicCountTokens)
792            .await
793        {
794            Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
795            Err(RunnerError::AuxiliaryUnsupported) => error_response(
796                StatusCode::BAD_REQUEST,
797                "route has no Anthropic target for token counting",
798                "invalid_request_error",
799                "count_tokens_unsupported",
800            ),
801            Err(RunnerError::Client(error)) => client_error(&error),
802            Err(error) => server_error(error.to_string()),
803        },
804    )
805}
806
807async fn openai_responses_input_tokens(
808    State(state): State<ServerState>,
809    headers: HeaderMap,
810    body: std::result::Result<Json<Value>, JsonRejection>,
811) -> Response {
812    openai_responses_auxiliary(
813        state,
814        headers,
815        body,
816        AuxiliaryOperation::ResponsesInputTokens,
817    )
818    .await
819}
820
821async fn openai_responses_compact(
822    State(state): State<ServerState>,
823    headers: HeaderMap,
824    body: std::result::Result<Json<Value>, JsonRejection>,
825) -> Response {
826    openai_responses_auxiliary(state, headers, body, AuxiliaryOperation::ResponsesCompact).await
827}
828
829// Resolves route aliases and configured credentials before calling a model-bearing Responses API.
830async fn openai_responses_auxiliary(
831    state: ServerState,
832    headers: HeaderMap,
833    body: std::result::Result<Json<Value>, JsonRejection>,
834    operation: AuxiliaryOperation,
835) -> Response {
836    let body = match llm_json_body(body) {
837        Ok(body) => body,
838        Err((status, message)) => {
839            return render_error_response(
840                invalid_body_error(status, message),
841                WireFormat::OpenAiResponses,
842            );
843        }
844    };
845    let (route, request) = match resolve_route(
846        &state,
847        metadata_from_headers(headers),
848        body,
849        WireFormat::OpenAiResponses,
850    ) {
851        Ok(resolved) => resolved,
852        Err(response) => return render_error_response(response, WireFormat::OpenAiResponses),
853    };
854    let result = route.call_auxiliary(request, operation).await;
855    render_error_response(
856        match result {
857            Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
858            Err(RunnerError::AuxiliaryUnsupported) => error_response(
859                StatusCode::BAD_REQUEST,
860                "route has no OpenAI Responses target",
861                "invalid_request_error",
862                "responses_auxiliary_unsupported",
863            ),
864            Err(RunnerError::Client(error)) => client_error(&error),
865            Err(error) => server_error(error.to_string()),
866        },
867        WireFormat::OpenAiResponses,
868    )
869}
870
871async fn handle_endpoint(
872    state: ServerState,
873    started: RequestStart,
874    headers: HeaderMap,
875    body: std::result::Result<Json<Value>, JsonRejection>,
876    wire_format: WireFormat,
877) -> Response {
878    let span = observability::request_span(&headers);
879    handle_endpoint_inner(state, started, headers, body, wire_format)
880        .instrument(span)
881        .await
882}
883
884async fn handle_endpoint_inner(
885    state: ServerState,
886    started: RequestStart,
887    headers: HeaderMap,
888    body: std::result::Result<Json<Value>, JsonRejection>,
889    wire_format: WireFormat,
890) -> Response {
891    let metadata = metadata_from_headers(headers);
892    let routing_log_context = state
893        .routing_log
894        .as_ref()
895        .map(|_| routing_log::RoutingLogContext::from_metadata(&metadata));
896    let request_log = RequestLogContext {
897        started: started.0,
898        wire_format,
899        requested_model: body
900            .as_ref()
901            .ok()
902            .and_then(|body| body.0.get("model"))
903            .and_then(Value::as_str)
904            .map(str::to_string),
905        streaming: body
906            .as_ref()
907            .ok()
908            .and_then(|body| body.0.get("stream"))
909            .and_then(Value::as_bool)
910            .unwrap_or(false),
911        session_id: metadata.session_id.clone(),
912        correlation_id: metadata.correlation_id.clone(),
913    };
914
915    // Axum drops this future when the downstream client disconnects before the
916    // response is written, which cancels the run and its upstream calls. The guard
917    // keeps that terminal state observable: the `emit` path below never runs for an
918    // abandoned buffered request.
919    let mut request_log = RequestLogGuard(Some(request_log));
920
921    let response = match llm_json_body(body) {
922        Ok(body) => {
923            handle_llm_request(
924                state,
925                started,
926                metadata,
927                body,
928                wire_format,
929                routing_log_context,
930            )
931            .await
932        }
933        Err((status, message)) => invalid_body_error(status, message),
934    };
935    let response = render_error_response(response, wire_format);
936    metrics::record_client_response(response.status().as_u16());
937    request_log.emit(&response);
938    response
939}
940
941fn llm_json_body(
942    body: std::result::Result<Json<Value>, JsonRejection>,
943) -> std::result::Result<Value, (StatusCode, String)> {
944    match body {
945        Ok(Json(value)) if value.is_object() => Ok(value),
946        Ok(_) => Err((
947            StatusCode::BAD_REQUEST,
948            "Request body must be a JSON object".to_string(),
949        )),
950        Err(error) => Err((
951            error.status(),
952            format!("Request body must be valid JSON: {error}"),
953        )),
954    }
955}
956
957/// Decode `body`, resolve the route named by its `model`, and build the
958/// [`Request`]. Shared by the completion and `count_tokens` handlers. Returns
959/// the resolved route and the built request — or an error [`Response`]
960/// (invalid body, empty `model` → 400, unknown route → 404).
961// Both callers immediately return the `Err(Response)` as the HTTP response, so
962// the large error type is intentional, not propagated up a call stack.
963#[allow(clippy::type_complexity, clippy::result_large_err)]
964fn resolve_route(
965    state: &ServerState,
966    metadata: Metadata,
967    mut body: Value,
968    wire_format: WireFormat,
969) -> std::result::Result<(&Route, Request), Response> {
970    // Only trusted translation hops may supply exact request preservation state.
971    // Strip it before decoding and retaining the raw body for upstream replay.
972    if let Some(metadata) = body.get_mut("metadata").and_then(Value::as_object_mut) {
973        metadata.remove(switchyard_translation::util::SWITCHYARD_METADATA_KEY);
974    }
975    let llm_request = decode_request(wire_format, &body)
976        .map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?;
977    let requested_model = llm_request
978        .model
979        .clone()
980        .filter(|model| !model.trim().is_empty())
981        .ok_or_else(|| {
982            error_response(
983                StatusCode::BAD_REQUEST,
984                "request body must include a non-empty string `model`",
985                "invalid_request_error",
986                "invalid_request_error",
987            )
988        })?;
989    let route = state.route_for_model(&requested_model).ok_or_else(|| {
990        error_response(
991            StatusCode::NOT_FOUND,
992            format!("No route registered for model {requested_model}"),
993            "model_not_found",
994            "model_not_found",
995        )
996    })?;
997    if let Err(RunnerError::IncompatibleCallerFormat(caller_auth)) =
998        route.check_caller_format(wire_format)
999    {
1000        let (provider, expected_endpoint) = match caller_auth {
1001            CallerAuthKind::Anthropic => ("Anthropic", "/v1/messages"),
1002            CallerAuthKind::OpenAi => ("OpenAI", "/v1/chat/completions or /v1/responses"),
1003        };
1004        return Err(error_response(
1005            StatusCode::BAD_REQUEST,
1006            format!(
1007                "route {requested_model} forwards an {provider} login; call it through {expected_endpoint}",
1008            ),
1009            "invalid_request_error",
1010            "invalid_request_error",
1011        ));
1012    }
1013    if let Some(capability) =
1014        capabilities::unsupported_capability(route.capabilities(), &llm_request, &body)
1015    {
1016        return Err(error_response(
1017            StatusCode::BAD_REQUEST,
1018            format!(
1019                "route {requested_model} declares {capability} = false; remove the unsupported input or select another route"
1020            ),
1021            "invalid_request_error",
1022            "unsupported_capability",
1023        ));
1024    }
1025    let request = Request {
1026        llm_request,
1027        raw_request: Some(body),
1028        metadata: Some(metadata),
1029    };
1030    Ok((route, request))
1031}
1032
1033/// Resolves and executes an LLM request, attaching route identity when durable logging is enabled.
1034async fn handle_llm_request(
1035    state: ServerState,
1036    started: RequestStart,
1037    metadata: Metadata,
1038    body: Value,
1039    wire_format: WireFormat,
1040    routing_log_context: Option<routing_log::RoutingLogContext>,
1041) -> Response {
1042    let cache_probe = state.track_cache_eligibility.then(|| prefix_probe(&body));
1043    let (route, request) = match resolve_route(&state, metadata, body, wire_format) {
1044        Ok(resolved) => resolved,
1045        Err(response) => return response,
1046    };
1047    let routing_log_context = routing_log_context.map(|context| {
1048        context.with_route(
1049            request.llm_request.model.as_deref().unwrap_or_default(),
1050            route.algorithm_name(),
1051        )
1052    });
1053    // Only the Codex namespace mapping is needed downstream, not the whole request.
1054    let request_extensions = request.llm_request.extensions.clone();
1055    let observer = stats_observer(
1056        state.stats.clone(),
1057        state.routing_log.clone().zip(routing_log_context.clone()),
1058    );
1059
1060    let output = match route.execute(request, Some(observer)).await {
1061        Ok(output) => output,
1062        Err(error) => {
1063            if let RunnerError::Algorithm(LibsyError::ClientCall {
1064                target,
1065                source:
1066                    LlmClientError::ResponseStateLimitExceeded { .. }
1067                    | LlmClientError::ResponseStateConflict,
1068            }) = &error
1069            {
1070                state.stats.record_response_error(target);
1071            }
1072            return runner_error(error);
1073        }
1074    };
1075    let RunOutput {
1076        selected_model,
1077        response,
1078    } = output;
1079    // The response carries the candidate that actually served it. Fall back to the routing
1080    // selection for algorithms that return a response without an offloaded model call.
1081    let served_model = response.served_model().cloned().or(Some(selected_model));
1082    let mut response = if let Some(served_model) = served_model.as_ref() {
1083        let cache_eligible = cache_probe
1084            .as_ref()
1085            .map(|probe| state.stats.prefix_eligibility(served_model, probe))
1086            .unwrap_or(0.0);
1087        usage_metrics::observe(
1088            response,
1089            served_model.as_str(),
1090            started.0,
1091            state.stats,
1092            cache_eligible,
1093            state.routing_log.zip(routing_log_context),
1094        )
1095    } else {
1096        response
1097    };
1098
1099    let upstream_headers = std::mem::take(&mut response.upstream_headers);
1100    let response_model = served_model.as_ref().map(ToString::to_string);
1101    let mut response = match into_http_response(
1102        response,
1103        wire_format,
1104        response_model,
1105        request_extensions,
1106        Arc::clone(&state.redactor),
1107    ) {
1108        Ok(response) => response,
1109        Err(error) => return server_error(error.to_string()),
1110    };
1111    // Forward upstream headers before Switchyard writes its own so any header
1112    // this server emits always overrides an upstream echo of the same name.
1113    let response_headers = response.headers_mut();
1114    for (name, value) in upstream_headers.iter() {
1115        if !should_forward_upstream_header(name) {
1116            continue;
1117        }
1118        response_headers.append(name.clone(), value.clone());
1119    }
1120    if let Some(served_model) = served_model.as_ref() {
1121        attach_routing_headers(&mut response, served_model.as_str());
1122    }
1123    response
1124}
1125
1126/// Holds the request log context until the request reaches a terminal state.
1127/// Dropping the guard with the context still in place means the handler future was
1128/// cancelled — the client left before a response existed — so the run is reported
1129/// as abandoned rather than silently disappearing.
1130struct RequestLogGuard(Option<RequestLogContext>);
1131
1132impl RequestLogGuard {
1133    // Emits the normal terminal event and disarms the cancellation path.
1134    fn emit(&mut self, response: &Response) {
1135        if let Some(context) = self.0.take() {
1136            context.emit(response);
1137        }
1138    }
1139}
1140
1141impl Drop for RequestLogGuard {
1142    fn drop(&mut self) {
1143        if let Some(context) = self.0.take() {
1144            context.emit_cancelled();
1145        }
1146    }
1147}
1148
1149// Request metadata held until the terminal response determines the event level.
1150struct RequestLogContext {
1151    started: Instant,
1152    wire_format: WireFormat,
1153    requested_model: Option<String>,
1154    streaming: bool,
1155    session_id: Option<String>,
1156    correlation_id: Option<String>,
1157}
1158
1159// Error text carried separately so terminal logging never consumes an HTTP body.
1160#[derive(Clone)]
1161struct RequestLogError(String);
1162
1163impl RequestLogContext {
1164    fn emit(self, response: &Response) {
1165        let selected_model = response
1166            .headers()
1167            .get(HEADER_SELECTED_MODEL)
1168            .and_then(|value| value.to_str().ok())
1169            .unwrap_or("");
1170        let duration_ms = self.started.elapsed().as_secs_f64() * 1_000.0;
1171        let error = response
1172            .extensions()
1173            .get::<RequestLogError>()
1174            .map(|error| error.0.as_str())
1175            .unwrap_or("");
1176
1177        macro_rules! emit {
1178            ($level:expr, $message:literal) => {
1179                tracing::event!(
1180                    target: "switchyard_server::request",
1181                    $level,
1182                    wire_format = %self.wire_format,
1183                    status = response.status().as_u16(),
1184                    requested_model = self.requested_model.as_deref().unwrap_or(""),
1185                    selected_model,
1186                    streaming = self.streaming,
1187                    session_id = self.session_id.as_deref().unwrap_or(""),
1188                    correlation_id = self.correlation_id.as_deref().unwrap_or(""),
1189                    handling_duration_ms = duration_ms,
1190                    error,
1191                    $message
1192                )
1193            };
1194        }
1195
1196        match request_log_level(response.status()) {
1197            Level::ERROR => emit!(Level::ERROR, "LLM request failed"),
1198            Level::WARN => emit!(Level::WARN, "LLM request failed"),
1199            _ => emit!(Level::INFO, "LLM request handled"),
1200        }
1201    }
1202
1203    // Terminal event for a request whose client left before a response was written.
1204    // No model served it, so there is no selected model and no response status.
1205    fn emit_cancelled(self) {
1206        metrics::record_client_disconnect();
1207        tracing::event!(
1208            target: "switchyard_server::request",
1209            Level::WARN,
1210            wire_format = %self.wire_format,
1211            status = CLIENT_CLOSED_REQUEST,
1212            requested_model = self.requested_model.as_deref().unwrap_or(""),
1213            selected_model = "",
1214            streaming = self.streaming,
1215            session_id = self.session_id.as_deref().unwrap_or(""),
1216            correlation_id = self.correlation_id.as_deref().unwrap_or(""),
1217            handling_duration_ms = self.started.elapsed().as_secs_f64() * 1_000.0,
1218            error = "client disconnected before a response was written",
1219            "LLM request cancelled"
1220        );
1221    }
1222}
1223
1224fn request_log_level(status: StatusCode) -> Level {
1225    if status.is_server_error() {
1226        Level::ERROR
1227    } else if status.is_success() {
1228        Level::INFO
1229    } else {
1230        Level::WARN
1231    }
1232}
1233
1234fn metadata_from_headers(headers: HeaderMap) -> Metadata {
1235    let mut metadata = Metadata::from_headers(&headers);
1236    metadata.http_headers = Some(headers);
1237    metadata
1238}
1239
1240fn attach_routing_headers(response: &mut Response, served_model: &str) {
1241    insert_routing_header(response, HEADER_SELECTED_MODEL, served_model);
1242}
1243
1244fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) {
1245    let Some(value) = sanitize_routing_header_value(value) else {
1246        return;
1247    };
1248    let Ok(value) = HeaderValue::from_str(&value) else {
1249        return;
1250    };
1251    response
1252        .headers_mut()
1253        .insert(HeaderName::from_static(name), value);
1254}
1255
1256fn sanitize_routing_header_value(value: &str) -> Option<String> {
1257    let value = value.split_whitespace().collect::<Vec<_>>().join(" ");
1258    (!value.is_empty()).then(|| value.chars().take(MAX_ROUTING_HEADER_VALUE_LEN).collect())
1259}
1260
1261fn algorithm_error(error: LibsyError) -> Response {
1262    let LibsyError::ClientCall { source, .. } = &error else {
1263        return server_error(error.to_string());
1264    };
1265    client_error(source)
1266}
1267
1268fn runner_error(error: RunnerError) -> Response {
1269    match error {
1270        RunnerError::Algorithm(error) => algorithm_error(error),
1271        RunnerError::Client(error) => client_error(&error),
1272        error => server_error(error.to_string()),
1273    }
1274}
1275
1276fn client_error(error: &LlmClientError) -> Response {
1277    match error {
1278        LlmClientError::ResponseStateLimitExceeded { .. } => error_response(
1279            StatusCode::SERVICE_UNAVAILABLE,
1280            error.to_string(),
1281            "server_error",
1282            "response_state_limit_exceeded",
1283        ),
1284        LlmClientError::ResponseStateConflict => error_response(
1285            StatusCode::CONFLICT,
1286            error.to_string(),
1287            "server_error",
1288            "response_state_conflict",
1289        ),
1290        LlmClientError::InvalidRequest { message }
1291        | LlmClientError::RequestTranslation(message) => error_response(
1292            StatusCode::BAD_REQUEST,
1293            message,
1294            "invalid_request_error",
1295            "invalid_request_error",
1296        ),
1297        LlmClientError::Configuration { message } => error_response(
1298            StatusCode::BAD_GATEWAY,
1299            message,
1300            "upstream_error",
1301            "upstream_configuration_error",
1302        ),
1303        LlmClientError::ContextWindowExceeded { message, .. } => error_response(
1304            StatusCode::BAD_REQUEST,
1305            message,
1306            "invalid_request_error",
1307            "context_length_exceeded",
1308        ),
1309        LlmClientError::UpstreamHttp { status, body } => upstream_error(*status, body),
1310        LlmClientError::Transport { source } | LlmClientError::InvalidResponse { source } => {
1311            error_response(
1312                StatusCode::BAD_GATEWAY,
1313                source.to_string(),
1314                "upstream_error",
1315                "upstream_error",
1316            )
1317        }
1318        LlmClientError::ResponseTranslation(message) => error_response(
1319            StatusCode::BAD_GATEWAY,
1320            message,
1321            "upstream_error",
1322            "upstream_error",
1323        ),
1324        LlmClientError::Timeout { source } => error_response(
1325            StatusCode::GATEWAY_TIMEOUT,
1326            source.to_string(),
1327            "upstream_error",
1328            "upstream_timeout",
1329        ),
1330        LlmClientError::RequestEncoding(message) => server_error(message),
1331        _ => server_error(error.to_string()),
1332    }
1333}
1334
1335// Keep the provider's message and nonempty string code in our error JSON.
1336fn upstream_error(status: StatusCode, body: &str) -> Response {
1337    let parsed = serde_json::from_str::<Value>(body).unwrap_or_default();
1338    let error = &parsed["error"];
1339    let message = error["message"].as_str().unwrap_or(body);
1340    let code = error["code"]
1341        .as_str()
1342        .filter(|code| !code.is_empty())
1343        .unwrap_or("upstream_error");
1344    error_response(status, message, "upstream_error", code)
1345}
1346
1347// Keep error details until the endpoint chooses its response format.
1348#[derive(Clone)]
1349struct ApiError {
1350    status: StatusCode,
1351    message: String,
1352    error_type: &'static str,
1353    code: String,
1354}
1355
1356impl ApiError {
1357    fn new(
1358        status: StatusCode,
1359        message: impl Into<String>,
1360        error_type: &'static str,
1361        code: impl Into<String>,
1362    ) -> Self {
1363        Self {
1364            status,
1365            message: message.into(),
1366            error_type,
1367            code: code.into(),
1368        }
1369    }
1370
1371    fn into_response(self, wire_format: WireFormat) -> Response {
1372        let body = match wire_format {
1373            WireFormat::AnthropicMessages => json!({
1374                "type": "error",
1375                "error": {
1376                    "type": anthropic_error_type(self.status),
1377                    "message": self.message.clone(),
1378                }
1379            }),
1380            WireFormat::OpenAiChat | WireFormat::OpenAiResponses => json!({
1381                "error": {
1382                    "message": self.message.clone(),
1383                    "type": self.error_type,
1384                    "code": self.code,
1385                }
1386            }),
1387        };
1388        let mut response = (self.status, Json(body)).into_response();
1389        response
1390            .extensions_mut()
1391            .insert(RequestLogError(self.message.clone()));
1392        response.extensions_mut().insert(self);
1393        response
1394    }
1395}
1396
1397fn render_error_response(response: Response, wire_format: WireFormat) -> Response {
1398    let Some(error) = response.extensions().get::<ApiError>().cloned() else {
1399        return response;
1400    };
1401    error.into_response(wire_format)
1402}
1403
1404fn anthropic_error_response(response: Response) -> Response {
1405    render_error_response(response, WireFormat::AnthropicMessages)
1406}
1407
1408fn anthropic_error_type(status: StatusCode) -> &'static str {
1409    match status {
1410        StatusCode::BAD_REQUEST => "invalid_request_error",
1411        StatusCode::UNAUTHORIZED => "authentication_error",
1412        StatusCode::FORBIDDEN => "permission_error",
1413        StatusCode::NOT_FOUND => "not_found_error",
1414        StatusCode::PAYLOAD_TOO_LARGE => "request_too_large",
1415        StatusCode::TOO_MANY_REQUESTS => "rate_limit_error",
1416        status if status.as_u16() == 529 => "overloaded_error",
1417        _ => "api_error",
1418    }
1419}
1420
1421fn server_error(message: impl Into<String>) -> Response {
1422    error_response(
1423        StatusCode::INTERNAL_SERVER_ERROR,
1424        message,
1425        "server_error",
1426        "server_error",
1427    )
1428}
1429
1430fn invalid_body_error(status: StatusCode, message: impl Into<String>) -> Response {
1431    error_response(status, message, "invalid_request_error", "invalid_body")
1432}
1433
1434fn error_response(
1435    status: StatusCode,
1436    message: impl Into<String>,
1437    error_type: &'static str,
1438    code: impl Into<String>,
1439) -> Response {
1440    ApiError::new(status, message, error_type, code).into_response(WireFormat::OpenAiChat)
1441}
1442
1443async fn models(State(state): State<ServerState>) -> Json<Value> {
1444    Json(model_list_payload(
1445        state
1446            .runner
1447            .models()
1448            .map(|model| (model.id.as_str(), model.capabilities)),
1449    ))
1450}
1451
1452async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {
1453    Json(state.stats.snapshot())
1454}
1455
1456async fn reset_stats(State(state): State<ServerState>) -> Json<Value> {
1457    state.stats.reset();
1458    Json(json!({"status": "reset"}))
1459}
1460
1461#[derive(Deserialize)]
1462struct SessionStatsQuery {
1463    session_id: String,
1464}
1465
1466// TODO: This loads the entire file. It should stream the JSONL instead.
1467// Huge files will crash the demo server.
1468async fn get_session_stats(
1469    State(state): State<ServerState>,
1470    query: std::result::Result<Query<SessionStatsQuery>, QueryRejection>,
1471) -> Response {
1472    let Query(query) = match query {
1473        Ok(query) => query,
1474        Err(error) => {
1475            return error_response(
1476                StatusCode::BAD_REQUEST,
1477                error.to_string(),
1478                "invalid_request_error",
1479                "invalid_query",
1480            );
1481        }
1482    };
1483    let Some(routing_log) = state.routing_log else {
1484        // Should be unreachable
1485        return not_found().await;
1486    };
1487    let session_id = query.session_id.clone();
1488    // Loading and de-serializing a large file is a time consuming blocking operation
1489    let snapshot =
1490        match task::spawn_blocking(move || routing_log.snapshot_session(&session_id)).await {
1491            Ok(s) => s,
1492            Err(err) => {
1493                return server_error(format!("failed to snapshot: {err}"));
1494            }
1495        };
1496
1497    match snapshot {
1498        Ok(Some(snapshot)) => (StatusCode::OK, Json(snapshot)).into_response(),
1499        Ok(None) => error_response(
1500            StatusCode::NOT_FOUND,
1501            "Routing session not found",
1502            "not_found",
1503            "routing_session_not_found",
1504        ),
1505        Err(error) => server_error(format!("failed to read routing log: {error}")),
1506    }
1507}
1508
1509async fn health() -> Json<Value> {
1510    Json(json!({"status": "ok"}))
1511}
1512
1513async fn prometheus_metrics(State(state): State<ServerState>) -> Response {
1514    match metrics::encode(&state.metrics) {
1515        Ok(body) => ([(CONTENT_TYPE, metrics::CONTENT_TYPE)], body).into_response(),
1516        Err(error) => server_error(error),
1517    }
1518}
1519
1520async fn not_found() -> Response {
1521    error_response(
1522        StatusCode::NOT_FOUND,
1523        "Not Found",
1524        "not_found",
1525        "endpoint_not_found",
1526    )
1527}
1528
1529fn model_list_payload<'a>(
1530    entries: impl IntoIterator<Item = (&'a str, ModelCapabilities)>,
1531) -> Value {
1532    let mut entries = entries.into_iter().collect::<Vec<_>>();
1533    entries.sort_unstable_by_key(|(model_id, _)| *model_id);
1534    let model_ids = entries.iter().map(|(model, _)| *model).collect::<Vec<_>>();
1535    let first_id = model_ids.first().copied();
1536    let last_id = model_ids.last().copied();
1537    json!({
1538        "object": "list",
1539        "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::<Vec<_>>(),
1540        // Codex requires this key; an empty list preserves its own catalog and instructions.
1541        "models": [],
1542        "first_id": first_id,
1543        "last_id": last_id,
1544        "has_more": false,
1545        "default_model": first_id,
1546        "model_pool": model_ids,
1547    })
1548}
1549
1550fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {
1551    json!({
1552        "id": model,
1553        "object": "model",
1554        "type": "model",
1555        "created": 0,
1556        "owned_by": "switchyard",
1557        "display_name": model,
1558        "capabilities": {
1559            "streaming": true,
1560            "tool_calling": capabilities.tool_calling,
1561            "vision": capabilities.vision,
1562            "context_window": capabilities.context_window,
1563            "supported_inbound_formats": [
1564                "openai-chat-completions",
1565                "openai-responses",
1566                "anthropic-messages",
1567            ],
1568        },
1569    })
1570}
1571
1572fn startup_banner(options: &ServerRunOptions, state: &ServerState, color: bool) -> String {
1573    let scheme = if options.is_tls() { "https" } else { "http" };
1574    let listen_url = url_for_addr(scheme, options.addr);
1575    let request_url = request_url_for_addr(scheme, options.addr);
1576    let routes = state.models().collect::<Vec<_>>();
1577    let route_list = routes.join(", ");
1578    let example_model = routes.first().copied().unwrap_or("switchyard/route");
1579    let example_body = json!({
1580        "model": example_model,
1581        "messages": [{"role": "user", "content": "Hello from Switchyard"}],
1582    });
1583    let example_url = shell_quote(&format!("{request_url}/v1/chat/completions"));
1584    let example_body = shell_quote(&example_body.to_string());
1585    format!(
1586        "{}\nSwitchyard libsy server\n  listening: {}\n  routes: {}\n\nendpoints:\n{}\n\nexample:\n  curl -s {} \\\n    -H 'Content-Type: application/json' \\\n    -d {}",
1587        render_startup_banner_art(color),
1588        listen_url,
1589        route_list,
1590        endpoint_listing(state.routing_log.is_some()),
1591        example_url,
1592        example_body,
1593    )
1594}
1595
1596// Keep redirected logs plain. Terminal output applies NVIDIA-green ANSI truecolor per line.
1597fn render_startup_banner_art(color: bool) -> String {
1598    let banner = STARTUP_BANNER_ART.trim_end();
1599    if !color {
1600        return banner.to_string();
1601    }
1602
1603    let (red, green, blue) = (118, 185, 0);
1604    let mut rendered = String::new();
1605    for line in banner.lines() {
1606        rendered.push_str(&format!("\x1b[38;2;{red};{green};{blue}m{line}\x1b[0m\n"));
1607    }
1608    rendered.trim_end_matches('\n').to_string()
1609}
1610
1611fn dry_run_summary(state: &ServerState) -> String {
1612    format!(
1613        "server OK: {}",
1614        state.models().collect::<Vec<_>>().join(", ")
1615    )
1616}
1617
1618fn url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
1619    format!("{scheme}://{}:{}", host_for_url(addr.ip()), addr.port())
1620}
1621
1622// Use loopback in request examples when the listener binds all interfaces.
1623fn request_url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
1624    let host = if addr.ip().is_unspecified() {
1625        match addr.ip() {
1626            std::net::IpAddr::V4(_) => "127.0.0.1".to_string(),
1627            std::net::IpAddr::V6(_) => "[::1]".to_string(),
1628        }
1629    } else {
1630        host_for_url(addr.ip())
1631    };
1632    format!("{scheme}://{host}:{}", addr.port())
1633}
1634
1635// Bracket IPv6 literals so they are valid inside a URL authority.
1636fn host_for_url(ip: std::net::IpAddr) -> String {
1637    match ip {
1638        std::net::IpAddr::V4(ip) => ip.to_string(),
1639        std::net::IpAddr::V6(ip) => format!("[{ip}]"),
1640    }
1641}
1642
1643fn shell_quote(value: &str) -> String {
1644    format!("'{}'", value.replace('\'', "'\\''"))
1645}
1646
1647fn endpoint_listing(has_routing_log: bool) -> String {
1648    let mut endpoints = vec![
1649        "  POST /v1/chat/completions    OpenAI Chat Completions",
1650        "  POST /v1/messages            Anthropic Messages",
1651        "  POST /v1/responses           OpenAI Responses",
1652        "  POST /v1/messages/count_tokens",
1653        "  POST /v1/responses/input_tokens",
1654        "  POST /v1/responses/compact",
1655        "  ANY  unmatched paths          optional fallback client",
1656        "  GET  /v1/models              configured routes",
1657        "  GET  /v1/stats               routing stats",
1658        "  POST /v1/stats/reset",
1659        "  GET  /metrics                Prometheus metrics",
1660        "  GET  /health",
1661    ];
1662    if has_routing_log {
1663        endpoints.push("  GET  /v1/routing/session-stats");
1664    }
1665    endpoints.join("\n")
1666}
1667
1668#[cfg(test)]
1669mod tests {
1670    use switchyard_llm_client::LlmCallObservation;
1671    use tokio::io::{AsyncReadExt, AsyncWriteExt};
1672    use tokio::sync::{Notify, oneshot};
1673
1674    use super::*;
1675
1676    /// A successful judge call lands in the per-session routing snapshot under its
1677    /// model id with the classifier tier, while routed calls stay off the observer's
1678    /// log path — they are logged with terminal usage when the served response is
1679    /// observed, so an append here would double count them.
1680    #[test]
1681    fn stats_observer_logs_judge_calls_to_the_routing_log() {
1682        let dir = tempfile::tempdir().expect("temp dir");
1683        let log = SharedRoutingLog::new(dir.path().join("routing.jsonl")).expect("routing log");
1684        let mut headers = HeaderMap::new();
1685        headers.insert("proxy_x_session_id", "session-1".parse().expect("header"));
1686        let metadata = metadata_from_headers(headers);
1687        let context = routing_log::RoutingLogContext::from_metadata(&metadata);
1688        let observer = stats_observer(StatsAccumulator::default(), Some((log.clone(), context)));
1689
1690        let call = |model: &str, answer: bool| {
1691            let observation = LlmCallObservation {
1692                selected_model: ModelId::from(model),
1693                is_success: true,
1694                duration: Duration::from_millis(3),
1695                usage: Some(Usage {
1696                    input_tokens: Some(100),
1697                    output_tokens: Some(7),
1698                    ..Usage::default()
1699                }),
1700            };
1701            if answer {
1702                RunObservation::AnswerCall(observation)
1703            } else {
1704                RunObservation::LlmCall(observation)
1705            }
1706        };
1707        observer(call("judge-model", false));
1708        observer(call("routed-model", true));
1709
1710        let snapshot = log
1711            .snapshot_session("session-1")
1712            .expect("read log")
1713            .expect("session recorded");
1714        let snapshot = serde_json::to_value(&snapshot).expect("serializable snapshot");
1715        assert_eq!(snapshot["models"]["judge-model"]["calls"], 1);
1716        assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100);
1717        assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7);
1718        assert!(snapshot["models"].get("routed-model").is_none());
1719    }
1720
1721    #[derive(Clone)]
1722    struct ShutdownTestState {
1723        started: Arc<Notify>,
1724        release: Arc<Notify>,
1725    }
1726
1727    struct ShutdownTestServer {
1728        state: ShutdownTestState,
1729        shutdown: oneshot::Sender<()>,
1730        server: task::JoinHandle<ServerResult<()>>,
1731        request: task::JoinHandle<std::io::Result<Vec<u8>>>,
1732    }
1733
1734    async fn blocked_request(State(state): State<ShutdownTestState>) -> &'static str {
1735        state.started.notify_one();
1736        state.release.notified().await;
1737        "done"
1738    }
1739
1740    async fn raw_request(addr: SocketAddr) -> std::io::Result<Vec<u8>> {
1741        let mut stream = tokio::net::TcpStream::connect(addr).await?;
1742        stream
1743            .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
1744            .await?;
1745        let mut response = Vec::new();
1746        stream.read_to_end(&mut response).await?;
1747        Ok(response)
1748    }
1749
1750    fn shutdown_test_server(shutdown_timeout: Duration) -> ShutdownTestServer {
1751        let state = ShutdownTestState {
1752            started: Arc::new(Notify::new()),
1753            release: Arc::new(Notify::new()),
1754        };
1755        let router = Router::new()
1756            .route("/", get(blocked_request))
1757            .with_state(state.clone());
1758        let listener = bind_tcp_listener("127.0.0.1:0".parse().expect("valid address"), 16)
1759            .expect("listener binds");
1760        let addr = listener.local_addr().expect("listener has an address");
1761        let (shutdown, shutdown_receiver) = oneshot::channel();
1762        let server = tokio::spawn(serve(listener, router, shutdown_timeout, async move {
1763            let _ = shutdown_receiver.await;
1764        }));
1765        let request = tokio::spawn(raw_request(addr));
1766        ShutdownTestServer {
1767            state,
1768            shutdown,
1769            server,
1770            request,
1771        }
1772    }
1773
1774    // Active requests may finish within the grace period, while stuck requests are bounded.
1775    #[tokio::test]
1776    async fn shutdown_drains_until_configured_deadline() {
1777        let ShutdownTestServer {
1778            state,
1779            shutdown,
1780            mut server,
1781            request,
1782        } = shutdown_test_server(Duration::from_secs(1));
1783        state.started.notified().await;
1784        shutdown.send(()).expect("server receives shutdown");
1785        assert!(
1786            tokio::time::timeout(Duration::from_millis(25), &mut server)
1787                .await
1788                .is_err(),
1789            "server must wait for the active request"
1790        );
1791        state.release.notify_one();
1792        tokio::time::timeout(Duration::from_secs(1), server)
1793            .await
1794            .expect("server stops after request drains")
1795            .expect("server task completes")
1796            .expect("server exits cleanly");
1797        let response = request
1798            .await
1799            .expect("request task completes")
1800            .expect("request succeeds");
1801        assert!(response.windows(8).any(|part| part == b"200 OK\r\n"));
1802        assert!(response.ends_with(b"done"));
1803
1804        let ShutdownTestServer {
1805            state,
1806            shutdown,
1807            server,
1808            request,
1809        } = shutdown_test_server(Duration::from_millis(25));
1810        state.started.notified().await;
1811        shutdown.send(()).expect("server receives shutdown");
1812        tokio::time::timeout(Duration::from_secs(1), server)
1813            .await
1814            .expect("shutdown deadline is enforced")
1815            .expect("server task completes")
1816            .expect("server exits cleanly");
1817        state.release.notify_one();
1818        request.abort();
1819    }
1820
1821    #[derive(Clone)]
1822    struct CancelTestState {
1823        started: Arc<Notify>,
1824        cancelled: Arc<Notify>,
1825    }
1826
1827    // Signals when the handler future is dropped, standing in for the in-flight
1828    // upstream call that a cancelled run releases.
1829    struct CancelSentinel(Arc<Notify>);
1830
1831    impl Drop for CancelSentinel {
1832        fn drop(&mut self) {
1833            self.0.notify_one();
1834        }
1835    }
1836
1837    async fn never_responds(State(state): State<CancelTestState>) -> &'static str {
1838        let _sentinel = CancelSentinel(state.cancelled.clone());
1839        state.started.notify_one();
1840        std::future::pending::<()>().await;
1841        "unreachable"
1842    }
1843
1844    // A buffered request produces no response until the run finishes, so nothing is
1845    // written to the socket to reveal that the client left. The server must still drop
1846    // the handler future on disconnect, otherwise upstream work outlives its client.
1847    #[tokio::test]
1848    async fn client_disconnect_cancels_a_buffered_handler() {
1849        let state = CancelTestState {
1850            started: Arc::new(Notify::new()),
1851            cancelled: Arc::new(Notify::new()),
1852        };
1853        let router = Router::new()
1854            .route("/", post(never_responds))
1855            .with_state(state.clone());
1856        let listener = bind_tcp_listener("127.0.0.1:0".parse().expect("valid address"), 16)
1857            .expect("listener binds");
1858        let addr = listener.local_addr().expect("listener has an address");
1859        let (shutdown, shutdown_receiver) = oneshot::channel();
1860        let server = tokio::spawn(serve(
1861            listener,
1862            router,
1863            Duration::from_millis(25),
1864            async move {
1865                let _ = shutdown_receiver.await;
1866            },
1867        ));
1868
1869        let mut stream = tokio::net::TcpStream::connect(addr)
1870            .await
1871            .expect("client connects");
1872        stream
1873            .write_all(b"POST / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}")
1874            .await
1875            .expect("client sends a complete request");
1876        state.started.notified().await;
1877
1878        // The client goes away mid-run, before any response byte exists.
1879        drop(stream);
1880
1881        tokio::time::timeout(Duration::from_secs(5), state.cancelled.notified())
1882            .await
1883            .expect("handler future is dropped when the client disconnects");
1884
1885        let _ = shutdown.send(());
1886        let _ = tokio::time::timeout(Duration::from_secs(5), server).await;
1887    }
1888
1889    // Collects the terminal request events emitted while it is the active subscriber.
1890    #[derive(Clone, Default)]
1891    struct CapturedEvents(Arc<Mutex<Vec<(Level, String)>>>);
1892
1893    impl tracing_subscriber::Layer<tracing_subscriber::Registry> for CapturedEvents {
1894        fn on_event(
1895            &self,
1896            event: &tracing::Event<'_>,
1897            _ctx: tracing_subscriber::layer::Context<'_, tracing_subscriber::Registry>,
1898        ) {
1899            if event.metadata().target() != "switchyard_server::request" {
1900                return;
1901            }
1902            let mut message = String::new();
1903            event.record(
1904                &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
1905                    if field.name() == "message" {
1906                        message = format!("{value:?}");
1907                    }
1908                },
1909            );
1910            self.0.lock().push((*event.metadata().level(), message));
1911        }
1912    }
1913
1914    fn request_log_context() -> RequestLogContext {
1915        RequestLogContext {
1916            started: Instant::now(),
1917            wire_format: WireFormat::OpenAiChat,
1918            requested_model: Some("switchyard/route".to_string()),
1919            streaming: false,
1920            session_id: Some("session".to_string()),
1921            correlation_id: None,
1922        }
1923    }
1924
1925    fn captured_events(run: impl FnOnce()) -> Vec<(Level, String)> {
1926        use tracing_subscriber::layer::SubscriberExt;
1927
1928        let captured = CapturedEvents::default();
1929        let subscriber = tracing_subscriber::registry().with(captured.clone());
1930        tracing::subscriber::with_default(subscriber, run);
1931        captured.0.lock().clone()
1932    }
1933
1934    // A dropped handler future is the only trace an abandoned buffered request leaves,
1935    // so the guard has to turn it into a terminal event rather than logging nothing.
1936    #[test]
1937    fn dropping_the_request_log_guard_reports_a_cancelled_request() {
1938        let events = captured_events(|| {
1939            drop(RequestLogGuard(Some(request_log_context())));
1940        });
1941
1942        assert_eq!(events.len(), 1);
1943        assert_eq!(events[0].0, Level::WARN);
1944        assert!(events[0].1.contains("cancelled"), "{:?}", events[0].1);
1945    }
1946
1947    // A request that reached a response is not a cancellation, even though the guard
1948    // is still dropped afterwards.
1949    #[test]
1950    fn an_answered_request_reports_only_its_response() {
1951        let events = captured_events(|| {
1952            let mut guard = RequestLogGuard(Some(request_log_context()));
1953            guard.emit(&StatusCode::OK.into_response());
1954        });
1955
1956        assert_eq!(events.len(), 1);
1957        assert_eq!(events[0].0, Level::INFO);
1958        assert!(events[0].1.contains("handled"), "{:?}", events[0].1);
1959    }
1960
1961    // Terminal request severity follows HTTP status instead of error-path bookkeeping.
1962    #[test]
1963    fn request_log_level_follows_http_status() {
1964        assert_eq!(request_log_level(StatusCode::OK), Level::INFO);
1965        assert_eq!(request_log_level(StatusCode::BAD_REQUEST), Level::WARN);
1966        assert_eq!(
1967            request_log_level(StatusCode::INTERNAL_SERVER_ERROR),
1968            Level::ERROR
1969        );
1970    }
1971
1972    // Canonical error text remains available without consuming the response body.
1973    #[test]
1974    fn error_response_carries_request_log_error() {
1975        let response = error_response(
1976            StatusCode::BAD_REQUEST,
1977            "invalid request",
1978            "invalid_request_error",
1979            "invalid_request_error",
1980        );
1981
1982        assert_eq!(
1983            response
1984                .extensions()
1985                .get::<RequestLogError>()
1986                .map(|error| error.0.as_str()),
1987            Some("invalid request")
1988        );
1989    }
1990}