dynamo-llm 1.5.0

// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::{
    collections::{HashMap, HashSet},
    fmt::Display,
    sync::{Arc, LazyLock},
    time::{SystemTime, UNIX_EPOCH},
};

use axum::{
    Json, Router,
    body::Body,
    extract::State,
    http::Request,
    http::{HeaderMap, Method, StatusCode, Uri},
    middleware::{self, Next},
    response::{
        IntoResponse, Response,
        sse::{Event, KeepAlive, Sse},
    },
    routing::{get, post},
};
use base64::Engine as _;
use bytes::Bytes;
use dynamo_runtime::config::{env_is_truthy, environment_names::llm as env_llm};
use dynamo_runtime::{
    pipeline::{AsyncEngineContextProvider, Context},
    protocols::annotated::AnnotationsProvider,
};
use futures::{StreamExt, stream};
use http_body_util::LengthLimitError;
use serde::{Deserialize, Serialize, de::DeserializeOwned};

use super::{
    RouteDoc, apply_request_tool_call_parsing_options,
    disconnect::{
        ConnectionHandle, StreamErrorSignal, create_connection_monitor, monitor_for_disconnects,
        monitor_for_disconnects_with_activity, monitor_for_disconnects_with_error_signal,
    },
    error::{HttpError, invalid_argument},
    metadata::{attach_x_request_id, extract_metadata_from_http},
    metrics::{
        CancellationLabels, Endpoint, ErrorType, EventConverter,
        process_chat_response_and_observe_metrics,
        process_chat_response_using_event_converter_and_observe_metrics,
        process_response_and_observe_metrics,
        process_response_using_event_converter_and_observe_metrics,
    },
    service_v2,
};
use crate::engines::ValidateRequest;
use crate::preprocessor::{PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY, decode_base64_to_floats};
use crate::protocols::common::extensions::{
    AGENT_CONTEXT_CONTEXT_KEY, AgentContext, InputTrigger, NvExt as CommonNvExt,
    SESSION_AFFINITY_CONTEXT_KEY, SessionAffinityId, agent_context_from_headers,
    apply_frontend_nvext_policy, has_non_cache_salt_routing_headers, session_affinity_from_headers,
};
use crate::protocols::common::input_trigger::{
    classify_chat_request, classify_completion_request, classify_response_request,
};
use crate::protocols::openai::chat_completions::aggregator::ChatCompletionAggregator;
use crate::protocols::openai::{
    ParsingOptions,
    audios::{NvAudioSpeechResponse, NvCreateAudioSpeechRequest},
    chat_completions::{
        NvCreateChatCompletionRequest, NvCreateChatCompletionResponse,
        NvCreateChatCompletionStreamResponse,
    },
    classify::{NvCreateClassifyRequest, NvCreateClassifyResponse},
    completions::{NvCreateCompletionRequest, NvCreateCompletionResponse},
    delta_common,
    embeddings::{NvCreateEmbeddingRequest, NvCreateEmbeddingResponse},
    images::{NvCreateImageRequest, NvImagesResponse},
    pooling::{
        NvCreatePoolingRequest, NvCreatePoolingResponse, PoolingEmbedDType, PoolingEncodingFormat,
        PoolingEndianness, PoolingOutput,
    },
    responses::{
        NvCreateResponse, NvResponse, ResponseParams, ResponsesConversionError,
        chat_completion_to_response,
    },
    videos::{NvCreateVideoRequest, NvVideosResponse},
};
use crate::protocols::unified::UnifiedRequest;
use crate::request_template::{RequestTemplate, resolve_request_model};
use crate::types::Annotated;
use dynamo_protocols::types::ChatCompletionMessageContent;
use dynamo_protocols::types::ChatCompletionMessageToolCallChunk;
use dynamo_protocols::types::ChatCompletionStreamResponseDelta;
use dynamo_protocols::types::Choice;
use dynamo_protocols::types::responses::{
    CountInputTokensRequest, CountInputTokensResponse, ErrorObject,
};
use dynamo_runtime::logging::get_distributed_tracing_context;
use tracing::Instrument;

pub const DYNAMO_REQUEST_ID_HEADER: &str = "x-dynamo-request-id";

/// Dynamo Annotation for the request ID
pub const ANNOTATION_REQUEST_ID: &str = "request_id";

const VALIDATION_PREFIX: &str = "Validation: ";
const BATCH_FILE_STORAGE_NOT_IMPLEMENTED: &str = "Batch file storage is not implemented yet.";
const BATCH_JOB_STATE_NOT_IMPLEMENTED: &str =
    "Batch job lifecycle persistence is not implemented yet.";
const BATCH_OUTPUT_RETRIEVAL_NOT_IMPLEMENTED: &str =
    "Batch output file retrieval is not implemented yet.";

static FORCE_INCLUDE_USAGE: LazyLock<bool> =
    LazyLock::new(|| env_is_truthy(env_llm::DYN_ENABLE_FORCE_INCLUDE_USAGE));

use super::error::{BackendStatusAction, SanitizedError, overload_status_code};

pub(super) fn rl_router(
    drt: Arc<dynamo_runtime::DistributedRuntime>,
) -> anyhow::Result<axum::Router> {
    let config = dynamo_rl::RlDiscoveryConfig::from_env(drt);
    let state = dynamo_rl::RlDiscoveryState::new_from_env(config);
    Ok(dynamo_rl::rl_router(state))
}

// Default axum max body limit without configuring is 2MB: https://docs.rs/axum/latest/axum/extract/struct.DefaultBodyLimit.html
/// Default body limit in bytes (45MB) to support 500k+ token payloads.
/// Can be configured at runtime using the DYN_HTTP_BODY_LIMIT_MB environment variable.
pub(super) fn get_body_limit() -> usize {
    std::env::var(env_llm::DYN_HTTP_BODY_LIMIT_MB)
        .ok()
        .and_then(|s| s.parse::<usize>().ok())
        .map(|mb| mb * 1024 * 1024)
        .unwrap_or(45 * 1024 * 1024)
}

pub type ErrorResponse = (StatusCode, Json<ErrorMessage>);

#[derive(Serialize, Deserialize, Debug)]
pub(crate) struct ErrorMessage {
    message: String,
    #[serde(rename = "type")]
    error_type: String,
    code: u16,
    #[serde(skip_serializing_if = "Option::is_none")]
    details: Option<Box<serde_json::Value>>,
    #[serde(skip)]
    metric_error_type: Option<ErrorType>,
}

impl ErrorMessage {
    pub(crate) fn message(&self) -> &str {
        &self.message
    }
}

fn map_error_code_to_error_type(code: StatusCode) -> String {
    // The configured overload code is checked before `canonical_reason()`, not
    // after. `DYN_HTTP_OVERLOAD_STATUS_CODE` accepts any status, and an IANA
    // registered one has a canonical reason that would otherwise win: set it to
    // 507 and a load-shed response reported itself as "Insufficient Storage".
    // 529 never showed that, because IANA does not register it and
    // `canonical_reason()` returns `None`.
    if code == overload_status_code() {
        return "Overloaded".to_string();
    }
    match code.canonical_reason() {
        Some(reason) => reason.to_string(),
        // 499 is not IANA-registered (nginx convention for client-closed-request),
        // so canonical_reason() returns None. Use the de facto standard name.
        None if code.as_u16() == 499 => "Client Closed Request".to_string(),
        None => "UnknownError".to_string(),
    }
}

/// `error_type` for a genuine 503 (readiness, model-unavailable, no routable
/// worker) that is not itself a load-shed rejection. `map_error_code_to_error_type`
/// cannot be reused here: it checks `code == overload_status_code()` first, and
/// when an operator configures `DYN_HTTP_OVERLOAD_STATUS_CODE=503` that check
/// would relabel every one of these unrelated 503s as "Overloaded".
fn unavailable_error_type() -> String {
    StatusCode::SERVICE_UNAVAILABLE
        .canonical_reason()
        .expect("503 is IANA-registered")
        .to_string()
}

/// `error_type` for a genuine 400 that is not a load-shed rejection. Same
/// reasoning as `unavailable_error_type`: `map_error_code_to_error_type`
/// checks `code == overload_status_code()` first, and an operator can
/// configure `DYN_HTTP_OVERLOAD_STATUS_CODE=400`, which would otherwise
/// label unsupported content "Overloaded" — telling clients to retry a
/// request that can never succeed.
fn bad_request_error_type() -> String {
    StatusCode::BAD_REQUEST
        .canonical_reason()
        .expect("400 is IANA-registered")
        .to_string()
}

/// `error_type` for a genuine 500 (unhandled panic, bug, misconfiguration)
/// that is not a load-shed rejection. Same reasoning as `unavailable_error_type`:
/// `map_error_code_to_error_type` checks `code == overload_status_code()`
/// first, and an operator can configure `DYN_HTTP_OVERLOAD_STATUS_CODE=500`,
/// which would otherwise relabel every internal error as "Overloaded".
fn internal_error_type() -> String {
    StatusCode::INTERNAL_SERVER_ERROR
        .canonical_reason()
        .expect("500 is IANA-registered")
        .to_string()
}

/// Classify error for metrics based on status code and message
fn classify_error_for_metrics(code: StatusCode, message: &str) -> ErrorType {
    // Same reason as `map_error_code_to_error_type`: the configured overload
    // code goes first. A registered status such as 507 matches an arm below and
    // would otherwise be counted as `Internal`, so a load shed would look like a
    // server fault on the dashboards.
    if code == overload_status_code() {
        return ErrorType::Overload;
    }
    match code {
        StatusCode::BAD_REQUEST => {
            // 400
            if message.starts_with("Validation:") {
                ErrorType::Validation
            } else {
                ErrorType::Internal
            }
        }
        StatusCode::NOT_FOUND => ErrorType::NotFound, // 404
        StatusCode::NOT_IMPLEMENTED => ErrorType::NotImplemented, // 501
        StatusCode::TOO_MANY_REQUESTS => ErrorType::Overload, // 429
        StatusCode::SERVICE_UNAVAILABLE => ErrorType::Unavailable, // 503
        StatusCode::INTERNAL_SERVER_ERROR => ErrorType::Internal, // 500
        _ if code.as_u16() == 529 => ErrorType::Overload, // 529
        _ if code.as_u16() == 499 => ErrorType::Cancelled, // 499 Client Closed Request
        _ if code.is_client_error() => ErrorType::Validation, // other 4xx
        _ => ErrorType::Internal,                     // everything else
    }
}

/// Extract ErrorType from ErrorResponse for metrics
fn extract_error_type_from_response(response: &ErrorResponse) -> ErrorType {
    response
        .1
        .metric_error_type
        .clone()
        .unwrap_or_else(|| classify_error_for_metrics(response.0, &response.1.message))
}

fn responses_conversion_error_response(error: anyhow::Error) -> ErrorResponse {
    const CONTEXT: &str = "Failed to convert responses request";

    match error.downcast_ref::<ResponsesConversionError>() {
        Some(ResponsesConversionError::InvalidArgument(message)) => ErrorMessage::from_anyhow(
            invalid_argument(format!("{CONTEXT}: {message}")).into(),
            CONTEXT,
        ),
        Some(ResponsesConversionError::UnsupportedContent(message)) => {
            ErrorMessage::unsupported_content_error(format!(
                "{VALIDATION_PREFIX}{CONTEXT}: {message}"
            ))
        }
        None => ErrorMessage::from_anyhow(error, CONTEXT),
    }
}

fn responses_error_code(status_code: StatusCode) -> &'static str {
    match status_code {
        StatusCode::TOO_MANY_REQUESTS => "rate_limit_exceeded",
        code if code.is_client_error() => "invalid_prompt",
        _ => "server_error",
    }
}

/// Match `InvalidArgument` at top-level OR under `Backend()`.
/// `py_err_to_dynamo` wraps Python `ValueError`/`TypeError` as
/// `Backend(InvalidArgument)`; both variants are 400-worthy.
pub(crate) fn find_invalid_argument_in_chain<'a>(
    err: &'a (dyn std::error::Error + 'static),
) -> Option<&'a dynamo_runtime::error::DynamoError> {
    use dynamo_runtime::error::{BackendError, ErrorType};
    let mut current = Some(err);
    while let Some(e) = current {
        if let Some(dynamo_err) = e.downcast_ref::<dynamo_runtime::error::DynamoError>()
            && matches!(
                dynamo_err.error_type(),
                ErrorType::InvalidArgument | ErrorType::Backend(BackendError::InvalidArgument)
            )
        {
            return Some(dynamo_err);
        }
        current = e.source();
    }
    None
}

fn find_queue_rejection_in_chain<'a>(
    err: &'a (dyn std::error::Error + 'static),
) -> Option<&'a dynamo_kv_router::scheduling::QueueRejection> {
    let mut current = Some(err);
    while let Some(error) = current {
        if let Some(rejection) =
            error.downcast_ref::<dynamo_kv_router::scheduling::QueueRejection>()
        {
            return Some(rejection);
        }
        current = error.source();
    }
    None
}

impl ErrorMessage {
    /// Not Found Error
    pub fn model_not_found() -> ErrorResponse {
        let code = StatusCode::NOT_FOUND;
        let error_type = map_error_code_to_error_type(code);
        (
            code,
            Json(ErrorMessage {
                message: "Model not found".to_string(),
                error_type,
                code: code.as_u16(),
                details: None,
                metric_error_type: None,
            }),
        )
    }

    /// Convert a ModelManagerError to the appropriate HTTP response.
    ///
    /// `ModelUnavailable` is the dispatch-time backstop for the same condition
    /// the readiness gate ([`check_model_serving_ready`]) catches up front — a
    /// registered model with no servable worker set (whichever role is missing).
    /// It returns the identical canonical 503 body so both code paths speak with
    /// one voice to the client.
    pub fn from_model_error(e: &crate::discovery::ModelManagerError) -> ErrorResponse {
        match e {
            crate::discovery::ModelManagerError::ModelUnavailable(model) => {
                Self::service_unavailable_with_body(model_not_ready_message(model))
            }
            _ => Self::model_not_found(),
        }
    }

    /// Service Unavailable
    /// This is returned when the service is live, but not ready.
    ///
    /// Always reports the plain "Service Unavailable" type and
    /// `ErrorType::Unavailable`, even when `DYN_HTTP_OVERLOAD_STATUS_CODE` is
    /// configured to 503 — `map_error_code_to_error_type` and
    /// `classify_error_for_metrics` would otherwise relabel this readiness
    /// failure as "Overloaded", though it has nothing to do with load
    /// shedding.
    pub fn _service_unavailable() -> ErrorResponse {
        let code = StatusCode::SERVICE_UNAVAILABLE;
        (
            code,
            Json(ErrorMessage {
                message: "Service is not ready".to_string(),
                error_type: unavailable_error_type(),
                code: code.as_u16(),
                details: None,
                metric_error_type: Some(ErrorType::Unavailable),
            }),
        )
    }

    /// Service Unavailable with a structured message body. Used by readiness
    /// reporting to distinguish "model registered but not ready" from generic
    /// "service not ready".
    ///
    /// See [`Self::_service_unavailable`] for why `error_type` and
    /// `metric_error_type` are set directly rather than derived from `code`.
    pub fn service_unavailable_with_body(message: String) -> ErrorResponse {
        let code = StatusCode::SERVICE_UNAVAILABLE;
        (
            code,
            Json(ErrorMessage {
                message,
                error_type: unavailable_error_type(),
                code: code.as_u16(),
                details: None,
                metric_error_type: Some(ErrorType::Unavailable),
            }),
        )
    }

    /// Internal Service Error
    /// Return this error when the service encounters an internal error.
    /// We should return a generic message to the client instead of the real error.
    /// Internal Services errors are the result of misconfiguration or bugs in the service.
    /// Always reports the plain "Internal Server Error" type and
    /// `ErrorType::Internal`, even when `DYN_HTTP_OVERLOAD_STATUS_CODE` is
    /// configured to 500 — see [`internal_error_type`] for why
    /// `map_error_code_to_error_type` cannot be reused here.
    pub fn internal_server_error(msg: &str) -> ErrorResponse {
        tracing::error!("Internal server error: {msg}");
        let code = StatusCode::INTERNAL_SERVER_ERROR;
        (
            code,
            Json(ErrorMessage {
                message: msg.to_string(),
                error_type: internal_error_type(),
                code: code.as_u16(),
                details: None,
                metric_error_type: Some(ErrorType::Internal),
            }),
        )
    }

    /// Internal Server Error with sanitized client message.
    /// Logs `details` server-side and returns only `public_msg` to the client.
    /// Use this whenever the detail could carry an anyhow chain, JoinError
    /// debug output, or anything else that may leak file paths, library
    /// versions, or other internal implementation details.
    ///
    /// See [`Self::internal_server_error`] for why `error_type` and
    /// `metric_error_type` are set directly rather than derived from `code`.
    pub fn internal_server_error_with_details(
        public_msg: &str,
        details: impl std::fmt::Display,
    ) -> ErrorResponse {
        tracing::error!("Internal server error: {public_msg}: {details}");
        let code = StatusCode::INTERNAL_SERVER_ERROR;
        (
            code,
            Json(ErrorMessage {
                message: public_msg.to_string(),
                error_type: internal_error_type(),
                code: code.as_u16(),
                details: None,
                metric_error_type: Some(ErrorType::Internal),
            }),
        )
    }

    /// Build a sanitized error response from a [`SanitizedError`] variant.
    /// The status, public message, and protocol error_type all come from
    /// the variant — call sites do not pass any of them as literals.
    /// Server-side `details` are logged alongside the canonical category;
    /// the client only ever sees the variant's public message.
    pub fn sanitized_with_details(
        err: SanitizedError,
        details: impl std::fmt::Display,
    ) -> ErrorResponse {
        let status = err.status();
        if err.log_as_error() {
            tracing::error!(status = %status, "{err}: {details}");
        } else {
            tracing::debug!(status = %status, "{err}: {details}");
        }
        // SanitizedError::Unavailable/Internal and SanitizedError::Overloaded
        // can carry the same StatusCode once an operator points
        // DYN_HTTP_OVERLOAD_STATUS_CODE at 503 or 500 (see
        // `unavailable_error_type`/`internal_error_type`), so the variant, not
        // just the status, decides error_type/metric_error_type here.
        let (error_type, metric_error_type) = match err {
            SanitizedError::Unavailable => (unavailable_error_type(), Some(ErrorType::Unavailable)),
            SanitizedError::Internal => (internal_error_type(), Some(ErrorType::Internal)),
            _ => (map_error_code_to_error_type(status), None),
        };
        (
            status,
            Json(ErrorMessage {
                message: err.to_string(),
                error_type,
                code: status.as_u16(),
                details: None,
                metric_error_type,
            }),
        )
    }

    /// Answer 500, with the status the engine asserted in `details`.
    ///
    /// The number is all that crosses the boundary; the backend's own message
    /// stays server-side, because a 5xx body may carry filesystem paths.
    fn coerced_backend_error(
        asserted: StatusCode,
        details: impl std::fmt::Display,
    ) -> ErrorResponse {
        let (status, mut body) = ErrorMessage::sanitized_with_details(
            SanitizedError::Internal,
            format!("backend asserted status {}: {details}", asserted.as_u16()),
        );
        body.0.details = Some(Box::new(
            serde_json::json!({ "backend_status": asserted.as_u16() }),
        ));
        (status, body)
    }

    /// Not Implemented Error
    /// Return this error when the client requests a feature that is not yet implemented.
    /// This should be used for features that are planned but not available.
    pub fn not_implemented_error<T: Display>(msg: T) -> ErrorResponse {
        tracing::error!("Not Implemented error: {msg}");
        let code = StatusCode::NOT_IMPLEMENTED;
        let error_type = map_error_code_to_error_type(code);
        (
            code,
            Json(ErrorMessage {
                message: msg.to_string(),
                error_type,
                code: code.as_u16(),
                details: None,
                metric_error_type: None,
            }),
        )
    }

    /// Unsupported multimodal content is a client error: no retry can make the
    /// request succeed, and infrastructure above the frontend counts 5xx as a
    /// server-side fault. Answered 400 where `not_implemented_error` answers 501.
    pub fn unsupported_content_error<T: Display>(msg: T) -> ErrorResponse {
        tracing::debug!("Unsupported Content error: {msg}");
        let code = StatusCode::BAD_REQUEST;
        let error_type = bad_request_error_type();
        (
            code,
            Json(ErrorMessage {
                message: msg.to_string(),
                error_type,
                code: code.as_u16(),
                details: None,
                metric_error_type: Some(ErrorType::NotImplemented),
            }),
        )
    }

    pub fn request_headers_too_large(msg: &str) -> ErrorResponse {
        let code = StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE;
        let error_type = map_error_code_to_error_type(code);
        (
            code,
            Json(ErrorMessage {
                message: msg.to_string(),
                error_type,
                code: code.as_u16(),
                details: None,
                metric_error_type: None,
            }),
        )
    }

    /// The OAI endpoints call an [`dynamo.runtime::engine::AsyncEngine`] which are specialized to return
    /// an [`anyhow::Error`]. This method will convert the [`anyhow::Error`] into an [`HttpError`].
    /// If successful, it will return the [`HttpError`] as an [`ErrorMessage::internal_server_error`]
    /// with the details of the error.
    pub fn from_anyhow(err: anyhow::Error, alt_msg: &str) -> ErrorResponse {
        if let Some(rejection) = find_queue_rejection_in_chain(err.as_ref()) {
            let code = overload_status_code();
            return (
                code,
                Json(ErrorMessage {
                    message: rejection.to_string(),
                    error_type: map_error_code_to_error_type(code),
                    code: code.as_u16(),
                    details: serde_json::to_value(rejection).ok().map(Box::new),
                    metric_error_type: None,
                }),
            );
        }

        // Check for ResourceExhausted anywhere in the error chain → HTTP 529
        if super::metrics::request_was_rejected(err.as_ref()) {
            return ErrorMessage::sanitized_with_details(
                SanitizedError::Overloaded,
                format!("{err:#}"),
            );
        }

        // No backend workers are currently routable → HTTP 503.
        if super::metrics::request_was_unavailable(err.as_ref()) {
            return ErrorMessage::sanitized_with_details(
                SanitizedError::Unavailable,
                format!("{err:#}"),
            );
        }

        // InvalidArgument (top-level OR Backend) → 400.
        if let Some(dynamo_err) = find_invalid_argument_in_chain(err.as_ref()) {
            // The message may be an [`ErrorPayload`] envelope rather than prose,
            // so unwrap it; otherwise the client is shown the raw JSON. An
            // explicit client-error status inside the envelope (for example 415)
            // is honoured, matching what the in-stream path already does. A 5xx
            // is not, because reaching this arm means the worker classified the
            // failure as a request problem, and a 5xx body here would bypass the
            // sanitizing the other arms apply.
            let (message, explicit_status) =
                match serde_json::from_str::<ErrorPayload>(dynamo_err.message()) {
                    Ok(envelope) => {
                        let explicit_status = envelope
                            .code
                            .and_then(|code| StatusCode::from_u16(code).ok())
                            .filter(StatusCode::is_client_error);
                        (
                            envelope
                                .message
                                .unwrap_or_else(|| dynamo_err.message().to_string()),
                            explicit_status,
                        )
                    }
                    Err(_) => (dynamo_err.message().to_string(), None),
                };
            let code = explicit_status.unwrap_or(StatusCode::BAD_REQUEST);
            // An explicit status the worker asserted goes through the shared
            // policy, so a 499 answers with the same sanitized cancellation
            // body as every other HTTP path rather than the worker's own text,
            // which can name a context id or an internal file. A 400 remains a
            // validation error, even when the configured overload status also
            // happens to be 400.
            if let Some(explicit_status) = explicit_status
                && explicit_status != StatusCode::BAD_REQUEST
                && let BackendStatusAction::Sanitize(variant) =
                    BackendStatusAction::triage(explicit_status)
            {
                return ErrorMessage::sanitized_with_details(variant, message);
            }
            return (
                code,
                Json(ErrorMessage {
                    message,
                    error_type: if code == StatusCode::BAD_REQUEST {
                        bad_request_error_type()
                    } else {
                        map_error_code_to_error_type(code)
                    },
                    code: code.as_u16(),
                    details: None,
                    // A plain 400 keeps the validation override: a worker's
                    // refusal text does not carry the `Validation:` prefix
                    // `classify_error_for_metrics` looks for, so the request
                    // error would otherwise be counted as `Internal`. A status
                    // the envelope preserved classifies from that status
                    // instead, so a backend rate limit counts as `Overload`.
                    metric_error_type: (code == StatusCode::BAD_REQUEST)
                        .then_some(ErrorType::Validation),
                }),
            );
        }

        // Check for Cancelled anywhere in the error chain → HTTP 499 (Client Closed Request)
        if super::metrics::request_was_cancelled(err.as_ref()) {
            return ErrorMessage::sanitized_with_details(
                SanitizedError::Cancelled,
                format!("{err:#}"),
            );
        }

        // Then check for HttpError
        match err.downcast::<HttpError>() {
            Ok(http_error) => ErrorMessage::from_http_error(http_error),
            Err(err) => {
                ErrorMessage::internal_server_error_with_details(alt_msg, format!("{err:#}"))
            }
        }
    }

    /// Convert a backend-supplied [`HttpError`] into a client response.
    ///
    /// Parse first, so a code outside the HTTP status space cannot reach the
    /// response, then let [`BackendStatusAction::triage`] decide. A 5xx keeps
    /// its own status only when it is 503 or the configured overload code,
    /// which is what makes a deliberate load shed distinguishable from an
    /// internal error. The body text is sanitized either way.
    pub fn from_http_error(err: HttpError) -> ErrorResponse {
        let Ok(status) = StatusCode::from_u16(err.code) else {
            return ErrorMessage::sanitized_with_details(SanitizedError::Internal, err.message);
        };
        match BackendStatusAction::triage(status) {
            BackendStatusAction::Sanitize(variant) => {
                ErrorMessage::sanitized_with_details(variant, err.message)
            }
            BackendStatusAction::CoerceToInternal(asserted) => {
                ErrorMessage::coerced_backend_error(asserted, err.message)
            }
            // 4xx (non-499): forward the backend's own message.
            BackendStatusAction::ForwardClientError => (
                status,
                Json(ErrorMessage {
                    message: err.message,
                    error_type: map_error_code_to_error_type(status),
                    code: status.as_u16(),
                    details: None,
                    metric_error_type: None,
                }),
            ),
        }
    }
}

impl From<HttpError> for ErrorMessage {
    fn from(err: HttpError) -> Self {
        ErrorMessage {
            message: err.message,
            error_type: map_error_code_to_error_type(
                StatusCode::from_u16(err.code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
            ),
            code: err.code,
            details: None,
            metric_error_type: None,
        }
    }
}

// Problem: Currently we are using JSON from axum as the request validator. Whenever there is an invalid JSON, it will return a 422.
// But all the downstream apps that relies on openai based APIs, expects to get 400 for all these cases otherwise they fail badly
// Solution: Intercept the response from handlers and convert ANY 422 status codes to 400 with the actual error message.
pub async fn smart_json_error_middleware(request: Request<Body>, next: Next) -> Response {
    let response = next.run(request).await;

    if response.status() == StatusCode::UNPROCESSABLE_ENTITY {
        let (_parts, body) = response.into_parts();
        let body_bytes = axum::body::to_bytes(body, get_body_limit())
            .await
            .unwrap_or_default();
        let error_message = String::from_utf8_lossy(&body_bytes).to_string();
        (
            StatusCode::BAD_REQUEST,
            Json(ErrorMessage {
                message: error_message,
                error_type: map_error_code_to_error_type(StatusCode::BAD_REQUEST),
                code: StatusCode::BAD_REQUEST.as_u16(),
                details: None,
                metric_error_type: None,
            }),
        )
            .into_response()
    } else {
        // Pass through if it is not a 422
        response
    }
}

/// Return the request ID for the current request.
///
/// The canonical request ID is set by `make_inference_request_span()` and stored
/// in the `DistributedTraceContext` via `DistributedTraceIdLayer`. This function
/// retrieves it, falling back to a validated `x-dynamo-request-id` header value
/// (deprecated, DEP #7812) or a new UUID.
///
/// **Deprecation (DEP #7812):** The `x-dynamo-request-id` header is deprecated.
/// Clients should rely on server-generated request IDs instead of supplying their own.
pub(super) fn get_or_create_request_id(headers: &HeaderMap) -> String {
    // Validate x-dynamo-request-id header if present, warn on invalid values.
    // DEP #7812: x-dynamo-request-id is deprecated — clients should rely on
    // server-generated request IDs instead of supplying their own.
    let validated_header = if let Some(raw) = headers.get(DYNAMO_REQUEST_ID_HEADER) {
        tracing::warn!(
            "{} header is deprecated (DEP #7812); server-generated request IDs should be used instead",
            DYNAMO_REQUEST_ID_HEADER
        );
        match raw.to_str() {
            Err(_) => {
                tracing::warn!(
                    "{} header must be a valid UTF-8 string",
                    DYNAMO_REQUEST_ID_HEADER
                );
                None
            }
            Ok(s) if uuid::Uuid::parse_str(s).is_err() => {
                tracing::warn!(
                    "{} header must be a valid UUID, got: {}",
                    DYNAMO_REQUEST_ID_HEADER,
                    s
                );
                None
            }
            Ok(s) => Some(s.to_string()),
        }
    } else {
        None
    };

    // Prefer trace context (set by make_inference_request_span via DistributedTraceIdLayer)
    if let Some(trace_context) = get_distributed_tracing_context()
        && let Some(request_id) = trace_context.request_id
    {
        return request_id;
    }

    // Fallback: use validated header for backwards compat, or generate new UUID
    validated_header.unwrap_or_else(|| uuid::Uuid::new_v4().to_string())
}

pub(super) fn context_from_headers<T: Send + Sync + 'static>(
    request: T,
    request_id: String,
    headers: &HeaderMap,
) -> Result<Context<T>, ErrorResponse> {
    context_from_headers_with_input_trigger(request, request_id, headers, |_| None)
}

fn context_from_headers_with_input_trigger<T, F>(
    request: T,
    request_id: String,
    headers: &HeaderMap,
    classify_input_trigger: F,
) -> Result<Context<T>, ErrorResponse>
where
    T: Send + Sync + 'static,
    F: FnOnce(&T) -> Option<InputTrigger>,
{
    let metadata = extract_metadata_from_http(headers)
        .map_err(|err| ErrorMessage::request_headers_too_large(&err.to_string()))?;
    let mut request = Context::with_id_and_metadata(request, request_id, metadata);
    attach_x_request_id(&mut request, headers);
    if let Some(mut agent_context) = agent_context_from_headers(headers) {
        agent_context.input_trigger = classify_input_trigger(request.content());
        request.insert(AGENT_CONTEXT_CONTEXT_KEY, agent_context);
    }
    if let Some(session_affinity) = session_affinity_from_headers(headers) {
        request.insert(SESSION_AFFINITY_CONTEXT_KEY, session_affinity);
    }
    Ok(request)
}

fn copy_context_metadata<T: Send + Sync + 'static, U: Send + Sync + 'static>(
    source: &Context<T>,
    target: &mut Context<U>,
) {
    if crate::request_trace::is_enabled()
        && let Ok(x_request_id) =
            source.get::<String>(crate::request_trace::X_REQUEST_ID_CONTEXT_KEY)
    {
        target.insert(
            crate::request_trace::X_REQUEST_ID_CONTEXT_KEY,
            x_request_id.as_ref().clone(),
        );
    }

    if let Ok(agent_context) = source.get::<AgentContext>(AGENT_CONTEXT_CONTEXT_KEY) {
        target.insert(AGENT_CONTEXT_CONTEXT_KEY, agent_context.as_ref().clone());
    }
    if let Ok(session_affinity) = source.get::<SessionAffinityId>(SESSION_AFFINITY_CONTEXT_KEY) {
        target.insert(
            SESSION_AFFINITY_CONTEXT_KEY,
            session_affinity.as_ref().clone(),
        );
    }
}

/// Warn once when the disabled NvExt policy discards a field or routing header.
/// Honored cache salts and `x-tenant-id` headers do not cause this warning.
pub(super) fn warn_nvext_disabled(endpoint: &str, discarded: bool) {
    if discarded {
        tracing::warn!(
            endpoint,
            "request carried disabled nvext fields or routing headers; dropping them"
        );
    }
}

/// OpenAI Completions Request Handler
///
/// This method will handle the incoming request for the `/v1/completions endpoint`. The endpoint is a "source"
/// for an [`super::OpenAICompletionsStreamingEngine`] and will return a stream of
/// responses which will be forward to the client.
///
/// Note: For all requests, streaming or non-streaming, we always call the engine with streaming enabled. For
/// non-streaming requests, we will fold the stream into a single response as part of this handler.
async fn handler_completions(
    State(state): State<Arc<service_v2::State>>,
    headers: HeaderMap,
    body: Body,
) -> Result<Response, ErrorResponse> {
    let body = read_json_request_body(&headers, body).await?;
    let mut request: NvCreateCompletionRequest = parse_json_request("completions", &body)?;
    if *FORCE_INCLUDE_USAGE && request.inner.stream.unwrap_or(false) {
        delta_common::force_include_usage(&mut request.inner.stream_options);
    }

    // return a 503 if the service or model is not ready
    check_ready(&state)?;
    check_model_serving_ready(&state, &request.inner.model)?;

    if !state.nvext_enabled() {
        warn_nvext_disabled(
            "completions",
            request
                .nvext
                .as_ref()
                .is_some_and(CommonNvExt::has_non_cache_salt_fields)
                || has_non_cache_salt_routing_headers(&headers),
        );
    }
    request.nvext =
        apply_frontend_nvext_policy(request.nvext.take(), &headers, state.nvext_enabled());

    // create the context for the request
    let request_id = get_or_create_request_id(&headers);
    let streaming = request.inner.stream.unwrap_or(false);
    // Canonicalize alias → primary for the metric label.
    let canonical_model = state.manager().resolve_canonical_name(&request.inner.model);
    let cancellation_labels = CancellationLabels {
        model: state
            .manager()
            .metric_model_for(&canonical_model)
            .to_string(),
        endpoint: Endpoint::Completions.to_string(),
        request_type: if streaming { "stream" } else { "unary" }.to_string(),
    };
    let request =
        context_from_headers_with_input_trigger(request, request_id, &headers, |request| {
            Some(classify_completion_request(request))
        })?;
    let context = request.context();

    // create the connection handles
    let (mut connection_handle, stream_handle) = create_connection_monitor(
        context.clone(),
        Some(state.metrics_clone()),
        cancellation_labels,
    )
    .await;

    // possibly long running task
    // if this returns a streaming response, the stream handle will be armed and captured by the response stream
    let response = tokio::spawn(completions(state, request, stream_handle).in_current_span())
        .await
        .map_err(|e| {
            ErrorMessage::internal_server_error_with_details(
                "Failed to await chat completions task",
                format!("{e:?}"),
            )
        })?;

    // if we got here, then we will return a response and the potentially long running task has completed successfully
    // without need to be cancelled.
    connection_handle.disarm();

    response
}

#[tracing::instrument(skip_all)]
async fn completions(
    state: Arc<service_v2::State>,
    request: Context<NvCreateCompletionRequest>,
    stream_handle: ConnectionHandle,
) -> Result<Response, ErrorResponse> {
    use crate::protocols::openai::completions::get_prompt_batch_size;

    // return a 503 if the service is not ready
    check_ready(&state)?;

    // Validate stream_options is only used when streaming (NVBug 5662680)
    validate_completion_stream_options(&request)?;

    validate_completion_fields_generic(&request)?;

    // Detect batch prompts
    let batch_size = get_prompt_batch_size(&request.inner.prompt);
    let n = request.inner.n.unwrap_or(1);

    // If single prompt or single-element batch, use original flow
    if batch_size == 1 {
        return completions_single(state, request, stream_handle).await;
    }

    // Batch processing: handle multiple prompts
    completions_batch(state, request, stream_handle, batch_size, n).await
}

/// Handle single prompt completions (original logic)
#[tracing::instrument(skip_all)]
async fn completions_single(
    state: Arc<service_v2::State>,
    mut request: Context<NvCreateCompletionRequest>,
    stream_handle: ConnectionHandle,
) -> Result<Response, ErrorResponse> {
    let request_id = request.id().to_string();

    // todo - decide on default
    let streaming = request.inner.stream.unwrap_or(false);

    // todo - make the protocols be optional for model name
    // todo - when optional, if none, apply a default
    // Resolve an alias to its primary served name and rewrite the request so
    // engine routing, metrics, and the OpenAI response.model all use the
    // canonical primary (matching vLLM/SGLang, where an alias request still
    // responds with the primary served name). Non-aliases pass through, so
    // metric_model_for still applies its unknown-model cardinality guard.
    let canonical = state.manager().resolve_canonical_name(&request.inner.model);
    if canonical != request.inner.model {
        request.inner.model = canonical;
    }
    let model = request.inner.model.clone();
    let metric_model = state.manager().metric_model_for(&model).to_string();

    // Create inflight_guard early to ensure all errors are counted
    let mut inflight_guard = state.metrics_clone().create_inflight_guard(
        &metric_model,
        Endpoint::Completions,
        streaming,
        &request_id,
    );

    // Create http_queue_guard early - tracks time waiting to be processed
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    // todo - error handling should be more robust
    let (engine, parsing_options) = state
        .manager()
        .get_completions_engine_with_parsing(&model)
        .map_err(|e| {
            let err_response = ErrorMessage::from_model_error(&e);
            inflight_guard.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    let mut response_collector = state
        .metrics_clone()
        .create_response_collector(&metric_model);

    // prepare to process any annotations
    let annotations = request.annotations();

    // issue the generate call on the engine
    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model, super::metrics::Endpoint::Completions);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate completions");
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    // capture the context to cancel the stream if the client disconnects
    let ctx = stream.context();

    let annotations = annotations.map_or(Vec::new(), |annotations| {
        annotations
            .iter()
            .filter_map(|annotation| {
                if annotation == ANNOTATION_REQUEST_ID {
                    Annotated::<NvCreateCompletionResponse>::from_annotation(
                        ANNOTATION_REQUEST_ID,
                        &request_id,
                    )
                    .ok()
                } else {
                    None
                }
            })
            .collect::<Vec<_>>()
    });

    // apply any annotations to the front of the stream
    let stream = stream::iter(annotations).chain(stream);

    if streaming {
        // For streaming, we'll drop the http_queue_guard on the first token
        let mut http_queue_guard = Some(http_queue_guard);
        let stream = stream
            .filter(|r| {
                // Drop empty chunks from multi-byte token assembly
                futures::future::ready(
                    !r.data
                        .as_ref()
                        .is_some_and(is_empty_completion_stream_response),
                )
            })
            .map(move |response| {
                // Calls observe_response() on each token
                process_response_using_event_converter_and_observe_metrics(
                    EventConverter::from(response),
                    &mut response_collector,
                    &mut http_queue_guard,
                )
            })
            .filter_map(|result| {
                use futures::future;
                // Transpose Result<Option<T>> -> Option<Result<T>>
                future::ready(result.transpose())
            });
        let stream = monitor_for_disconnects(stream, ctx, inflight_guard, stream_handle);

        let mut sse_stream = Sse::new(stream);

        if let Some(keep_alive) = state.sse_keep_alive() {
            sse_stream = sse_stream.keep_alive(KeepAlive::default().interval(keep_alive));
        }

        Ok(sse_stream.into_response())
    } else {
        // Preserve typed backend errors before the completions aggregator turns
        // them into strings. In particular, Python ValueError/TypeError arrives
        // as Backend(InvalidArgument) and must remain an HTTP 400.
        let stream = check_for_backend_error(stream, None)
            .await
            .map_err(|error_response| {
                tracing::error!(request_id, "Backend error detected: {:?}", error_response);
                inflight_guard.mark_error(extract_error_type_from_response(&error_response));
                error_response
            })?;

        // Tap the stream to collect metrics for non-streaming requests without altering items
        let mut http_queue_guard = Some(http_queue_guard);
        let stream = stream.inspect(move |response| {
            // Calls observe_response() on each token - drops http_queue_guard on first token
            process_response_and_observe_metrics(
                response,
                &mut response_collector,
                &mut http_queue_guard,
            );
        });

        let response = NvCreateCompletionResponse::from_annotated_stream(stream, parsing_options)
            .await
            .map_err(|e| {
                tracing::error!(
                    "Failed to fold completions stream for {}: {:?}",
                    request_id,
                    e
                );
                let err_response = ErrorMessage::internal_server_error(&format!(
                    "Failed to fold completions stream for {request_id}"
                ));
                inflight_guard.mark_error(extract_error_type_from_response(&err_response));
                err_response
            })?;

        inflight_guard.mark_ok();
        // If the engine context was killed (client disconnect), the response was
        // assembled but never delivered. Override to cancelled.
        if ctx.is_killed() {
            inflight_guard.mark_error(ErrorType::Cancelled);
        }
        Ok(Json(response).into_response())
    }
}

fn add_optional_token_count(total: &mut Option<u32>, value: Option<u32>) {
    if let Some(value) = value {
        *total = Some(total.unwrap_or_default().saturating_add(value));
    }
}

fn merge_completion_usage(
    total: &mut dynamo_protocols::types::CompletionUsage,
    usage: dynamo_protocols::types::CompletionUsage,
) {
    total.prompt_tokens = total.prompt_tokens.saturating_add(usage.prompt_tokens);
    total.completion_tokens = total
        .completion_tokens
        .saturating_add(usage.completion_tokens);
    total.total_tokens = total.total_tokens.saturating_add(usage.total_tokens);

    if let Some(details) = usage.prompt_tokens_details {
        let total_details = total.prompt_tokens_details.get_or_insert_default();
        add_optional_token_count(&mut total_details.audio_tokens, details.audio_tokens);
        add_optional_token_count(&mut total_details.cached_tokens, details.cached_tokens);
    }

    if let Some(details) = usage.completion_tokens_details {
        let total_details = total.completion_tokens_details.get_or_insert_default();
        add_optional_token_count(
            &mut total_details.accepted_prediction_tokens,
            details.accepted_prediction_tokens,
        );
        add_optional_token_count(&mut total_details.audio_tokens, details.audio_tokens);
        add_optional_token_count(
            &mut total_details.reasoning_tokens,
            details.reasoning_tokens,
        );
        add_optional_token_count(
            &mut total_details.rejected_prediction_tokens,
            details.rejected_prediction_tokens,
        );
    }
}

/// Combine the terminal usage-only chunks from per-prompt streams into one
/// request-level chunk. Continuous usage attached to content chunks passes
/// through unchanged because those values are cumulative snapshots.
fn aggregate_batch_completion_usage(
    stream: impl futures::Stream<Item = Annotated<NvCreateCompletionResponse>>,
    request_id: String,
) -> impl futures::Stream<Item = Annotated<NvCreateCompletionResponse>> {
    async_stream::stream! {
        let mut stream = Box::pin(stream);
        let mut aggregate_usage = dynamo_protocols::types::CompletionUsage::default();
        let mut final_usage_chunk = None;

        while let Some(mut response) = stream.next().await {
            let terminal_usage = response.data.as_mut().and_then(|data| {
                data.inner
                    .choices
                    .is_empty()
                    .then(|| data.inner.usage.take())
                    .flatten()
            });

            if let Some(usage) = terminal_usage {
                merge_completion_usage(&mut aggregate_usage, usage);
                final_usage_chunk = Some(response);
                continue;
            }

            yield response;
        }

        if let Some(mut response) = final_usage_chunk {
            if let Some(data) = response.data.as_mut() {
                data.inner.id = format!("cmpl-{request_id}");
                data.inner.usage = Some(aggregate_usage);
            }
            yield response;
        }
    }
}

type BoxedCompletionResponseStream =
    std::pin::Pin<Box<dyn futures::Stream<Item = Annotated<NvCreateCompletionResponse>> + Send>>;

/// Check each prompt stream before merging a non-streaming completion batch.
///
/// `select_all` cannot safely provide this check after merging because a normal
/// event from one prompt may arrive before a typed backend error from another.
/// Poll all streams concurrently so batch startup is not serialized.
async fn check_completion_batch_streams<S>(
    streams: Vec<S>,
) -> Result<Vec<BoxedCompletionResponseStream>, ErrorResponse>
where
    S: futures::Stream<Item = Annotated<NvCreateCompletionResponse>> + Send + 'static,
{
    futures::future::try_join_all(
        streams
            .into_iter()
            .map(|stream| check_for_backend_error(stream, None)),
    )
    .await
}

/// Handle batch prompt completions (multiple prompts with n choices each)
#[tracing::instrument(skip_all)]
async fn completions_batch(
    state: Arc<service_v2::State>,
    mut request: Context<NvCreateCompletionRequest>,
    stream_handle: ConnectionHandle,
    batch_size: usize,
    n: u8,
) -> Result<Response, ErrorResponse> {
    use crate::protocols::openai::completions::extract_single_prompt;
    use futures::stream::{self, StreamExt};

    let request_id = request.id().to_string();
    let streaming = request.inner.stream.unwrap_or(false);
    // Resolve alias → primary served name (see completions_single).
    let canonical = state.manager().resolve_canonical_name(&request.inner.model);
    if canonical != request.inner.model {
        request.inner.model = canonical;
    }
    let model = request.inner.model.clone();
    let metric_model = state.manager().metric_model_for(&model).to_string();

    // Create inflight_guard early to ensure all errors are counted
    let mut inflight_guard = state.metrics_clone().create_inflight_guard(
        &metric_model,
        Endpoint::Completions,
        streaming,
        &request_id,
    );

    // Create http_queue_guard early - tracks time waiting to be processed
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    let (engine, parsing_options) = state
        .manager()
        .get_completions_engine_with_parsing(&model)
        .map_err(|e| {
            let err_response = ErrorMessage::from_model_error(&e);
            inflight_guard.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    let mut response_collector = state
        .metrics_clone()
        .create_response_collector(&metric_model);

    // prepare to process any annotations
    let annotations = request.annotations();

    // Generate streams for each prompt in the batch
    let mut all_streams = Vec::new();
    let mut first_ctx = None;

    for prompt_idx in 0..batch_size {
        // Extract single prompt at this index
        let single_prompt = extract_single_prompt(&request.inner.prompt, prompt_idx);

        // Create a new request with this single prompt
        let mut single_request = request.content().clone();
        single_request.inner.prompt = single_prompt;

        // Generate unique request_id for each prompt: original_id-{prompt_idx}
        let unique_request_id = format!("{}-{}", request.id(), prompt_idx);
        let mut single_request_context = Context::with_id_and_metadata(
            single_request,
            unique_request_id,
            request.metadata().clone(),
        );
        copy_context_metadata(&request, &mut single_request_context);

        // Generate stream for this prompt
        let stream = engine.generate(single_request_context).await.map_err(|e| {
            if super::metrics::request_was_rejected(e.as_ref()) {
                state
                    .metrics_clone()
                    .inc_rejection(&model, super::metrics::Endpoint::Completions);
            }
            let err_response = ErrorMessage::from_anyhow(e, "Failed to generate completions");
            inflight_guard.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

        // Capture context from first stream
        if first_ctx.is_none() {
            first_ctx = Some(stream.context());
        }

        // Remap choice indices: choice.index += prompt_idx * n
        let prompt_idx_u32 = prompt_idx as u32;
        let n_u32 = n as u32;
        let remapped_stream = stream.map(move |mut response| {
            if let Some(ref mut data) = response.data {
                for choice in &mut data.inner.choices {
                    choice.index += prompt_idx_u32 * n_u32;
                }
            }
            response
        });

        all_streams.push(remapped_stream);
    }

    let all_streams: Vec<BoxedCompletionResponseStream> = if streaming {
        all_streams
            .into_iter()
            .map(|stream| Box::pin(stream) as BoxedCompletionResponseStream)
            .collect()
    } else {
        check_completion_batch_streams(all_streams)
            .await
            .map_err(|error_response| {
                tracing::error!(request_id, "Backend error detected: {:?}", error_response);
                inflight_guard.mark_error(extract_error_type_from_response(&error_response));
                error_response
            })?
    };

    // Merge all streams after every non-streaming prompt has passed its own
    // backend-error preflight.
    let merged_stream = stream::select_all(all_streams);
    let merged_stream = aggregate_batch_completion_usage(merged_stream, request_id.clone());

    // capture the context to cancel the stream if the client disconnects
    let ctx = first_ctx.expect("At least one stream should be generated");

    let annotations_vec = annotations.map_or(Vec::new(), |annotations| {
        annotations
            .iter()
            .filter_map(|annotation| {
                if annotation == ANNOTATION_REQUEST_ID {
                    Annotated::<NvCreateCompletionResponse>::from_annotation(
                        ANNOTATION_REQUEST_ID,
                        &request_id,
                    )
                    .ok()
                } else {
                    None
                }
            })
            .collect::<Vec<_>>()
    });

    // apply any annotations to the front of the stream
    let merged_stream = stream::iter(annotations_vec).chain(merged_stream);

    if streaming {
        // For streaming, we'll drop the http_queue_guard on the first token
        let mut http_queue_guard = Some(http_queue_guard);
        let stream = merged_stream
            .filter(|r| {
                // Drop empty chunks from multi-byte token assembly
                futures::future::ready(
                    !r.data
                        .as_ref()
                        .is_some_and(is_empty_completion_stream_response),
                )
            })
            .map(move |response| {
                // Calls observe_response() on each token
                process_response_using_event_converter_and_observe_metrics(
                    EventConverter::from(response),
                    &mut response_collector,
                    &mut http_queue_guard,
                )
            })
            .filter_map(|result| {
                use futures::future;
                // Transpose Result<Option<T>> -> Option<Result<T>>
                future::ready(result.transpose())
            });
        let stream = monitor_for_disconnects(stream, ctx, inflight_guard, stream_handle);

        let mut sse_stream = Sse::new(stream);

        if let Some(keep_alive) = state.sse_keep_alive() {
            sse_stream = sse_stream.keep_alive(KeepAlive::default().interval(keep_alive));
        }

        Ok(sse_stream.into_response())
    } else {
        // Tap the stream to collect metrics for non-streaming requests without altering items
        let mut http_queue_guard = Some(http_queue_guard);
        let stream = merged_stream.inspect(move |response| {
            // Calls observe_response() on each token - drops http_queue_guard on first token
            process_response_and_observe_metrics(
                response,
                &mut response_collector,
                &mut http_queue_guard,
            );
        });

        let response = NvCreateCompletionResponse::from_annotated_stream(stream, parsing_options)
            .await
            .map_err(|e| {
                tracing::error!(
                    "Failed to fold completions stream for {}: {:?}",
                    request_id,
                    e
                );
                let err_response = ErrorMessage::internal_server_error(&format!(
                    "Failed to fold completions stream for {request_id}"
                ));
                inflight_guard.mark_error(extract_error_type_from_response(&err_response));
                err_response
            })?;

        inflight_guard.mark_ok();
        // If the engine context was killed (client disconnect), the response was
        // assembled but never delivered. Override to cancelled.
        if ctx.is_killed() {
            inflight_guard.mark_error(ErrorType::Cancelled);
        }
        Ok(Json(response).into_response())
    }
}

#[tracing::instrument(skip_all)]
async fn embeddings(
    State(state): State<Arc<service_v2::State>>,
    headers: HeaderMap,
    Json(mut request): Json<NvCreateEmbeddingRequest>,
) -> Result<Response, ErrorResponse> {
    // return a 503 if the service or model is not ready
    check_ready(&state)?;
    check_model_serving_ready(&state, &request.inner.model)?;

    if !state.nvext_enabled() {
        warn_nvext_disabled(
            "embeddings",
            request
                .nvext
                .as_ref()
                .is_some_and(|nvext| nvext.annotations.is_some()),
        );
        request.nvext = None;
    }

    // Resolve alias → primary served name before wrapping the request, so
    // engine routing, metrics, and the response model all use the canonical
    // primary (see completions_single). `request` is still owned + mutable here.
    let canonical = state.manager().resolve_canonical_name(&request.inner.model);
    if canonical != request.inner.model {
        request.inner.model = canonical;
    }
    let request_id = get_or_create_request_id(&headers);
    let request = context_from_headers(request, request_id, &headers)?;
    let request_id = request.id().to_string();

    // The worker always emits base64-encoded vectors over NATS so we
    // avoid serializing/parsing a JSON float array on the internal hop.
    // If the client asked for float (the default), decode back at the
    // HTTP boundary so the public response shape matches their
    // ``encoding_format`` choice. See the PR description / DIS-2154 for
    // measured impact.
    // Borrow rather than move ``encoding_format`` out of ``request`` so the
    // request value remains intact for the later ``engine.generate(request)``
    // call below.
    let client_wants_float = !matches!(
        request.inner.encoding_format.as_ref(),
        Some(dynamo_protocols::types::EncodingFormat::Base64)
    );

    // Embeddings are typically not streamed, so we default to non-streaming
    let streaming = false;

    // todo - make the protocols be optional for model name
    // todo - when optional, if none, apply a default
    let model = &request.inner.model;
    let metric_model = state.manager().metric_model_for(model).to_string();

    // Start the embedding-specific latency timer. Distinct from
    // `request_duration` (which has 1..512s LLM-gen buckets); pooling-model
    // requests are sub-second and need finer-grained buckets to be useful for
    // SLO tracking. Only observed on the success path -- error/cancel paths
    // already increment requests_total with status=error.
    let embedding_start = std::time::Instant::now();

    // Create inflight_guard early to ensure all errors are counted
    let mut inflight = state.metrics_clone().create_inflight_guard(
        &metric_model,
        Endpoint::Embeddings,
        streaming,
        &request_id,
    );

    // Create http_queue_guard early - tracks time waiting to be processed
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    // todo - error handling should be more robust
    let engine = state.manager().get_embeddings_engine(model).map_err(|e| {
        let err_response = ErrorMessage::from_model_error(&e);
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    let mut response_collector = state
        .metrics_clone()
        .create_response_collector(&metric_model);
    let model_name = model.to_string();

    // issue the generate call on the engine
    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model_name, super::metrics::Endpoint::Embeddings);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate embeddings");
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    // Process stream to collect metrics and drop http_queue_guard on first token
    let mut http_queue_guard = Some(http_queue_guard);
    let stream = stream.inspect(move |response| {
        // Calls observe_response() on each token - drops http_queue_guard on first token
        process_response_and_observe_metrics(
            response,
            &mut response_collector,
            &mut http_queue_guard,
        );
    });

    // Embeddings are typically returned as a single response (non-streaming)
    // so we fold the stream into a single response
    let mut response = NvCreateEmbeddingResponse::from_annotated_stream(stream)
        .await
        .map_err(|e| {
            tracing::error!(
                "Failed to fold embeddings stream for {}: {:?}",
                request_id,
                e
            );
            let err_response =
                ErrorMessage::internal_server_error("Failed to fold embeddings stream");
            inflight.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    // Convert an optimized internal Base64 payload back to Float when the
    // client asked for float (or omitted the format, which defaults to float).
    if client_wants_float {
        for embedding_obj in response.inner.data.iter_mut() {
            if let dynamo_protocols::types::EmbeddingVector::Base64(s) = &embedding_obj.embedding {
                match decode_base64_to_floats(s) {
                    Ok(floats) => {
                        embedding_obj.embedding =
                            dynamo_protocols::types::EmbeddingVector::Float(floats);
                    }
                    Err(e) => {
                        tracing::error!(
                            "Failed to decode base64 embedding for request {}: {:?}",
                            request_id,
                            e
                        );
                        let err_response = ErrorMessage::internal_server_error(
                            "Failed to decode embedding payload",
                        );
                        inflight.mark_error(extract_error_type_from_response(&err_response));
                        return Err(err_response);
                    }
                }
            }
        }
    }

    state
        .metrics_clone()
        .observe_embedding_latency(&model_name, embedding_start.elapsed().as_secs_f64());
    inflight.mark_ok();
    Ok(Json(response).into_response())
}

#[tracing::instrument(skip_all)]
async fn classify(
    State(state): State<Arc<service_v2::State>>,
    headers: HeaderMap,
    Json(mut request): Json<NvCreateClassifyRequest>,
) -> Result<Response, ErrorResponse> {
    // return a 503 if the service or model is not ready
    check_ready(&state)?;
    check_model_serving_ready(&state, &request.model)?;

    if !state.nvext_enabled() {
        warn_nvext_disabled(
            "classify",
            request
                .nvext
                .as_ref()
                .is_some_and(|nvext| nvext.annotations.is_some()),
        );
        request.nvext = None;
    }

    // Resolve alias → primary served name before wrapping the request, so
    // engine routing, metrics, and the response model all use the canonical
    // primary (mirrors `embeddings` / `completions_single`).
    let canonical = state.manager().resolve_canonical_name(&request.model);
    if canonical != request.model {
        request.model = canonical;
    }
    let request_id = get_or_create_request_id(&headers);
    let request = context_from_headers(request, request_id, &headers)?;
    let request_id = request.id().to_string();

    // Classification, like embeddings, is a pooling task returned as a single
    // (non-streaming) response.
    let streaming = false;

    let model = &request.model;
    let metric_model = state.manager().metric_model_for(model).to_string();

    // Create inflight_guard early to ensure all errors (including validation)
    // are counted. Request validation runs after this point so a rejected
    // request still lands in `requests_total` with error_type=validation
    // (mirrors `chat_completions`).
    let mut inflight = state.metrics_clone().create_inflight_guard(
        &metric_model,
        Endpoint::Classify,
        streaming,
        &request_id,
    );

    // Marked as `Validation` explicitly rather than through
    // `extract_error_type_from_response`: that helper infers the type from the
    // message, and only a `VALIDATION_PREFIX`-prefixed 400 maps to
    // `Validation` (anything else falls back to `Internal`). These messages
    // stay verbatim vLLM-compatible, so the prefix is not an option here.
    if let Err(err_response) = validate_pooling_cache_salt(request.cache_salt.as_deref()) {
        inflight.mark_error(ErrorType::Validation);
        return Err(err_response);
    }

    // Create http_queue_guard early - tracks time waiting to be processed
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    let engine = state.manager().get_classify_engine(model).map_err(|e| {
        let err_response = ErrorMessage::from_model_error(&e);
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    let mut response_collector = state
        .metrics_clone()
        .create_response_collector(&metric_model);
    let model_name = model.to_string();

    // issue the generate call on the engine
    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model_name, super::metrics::Endpoint::Classify);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate classification");
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    // Process stream to collect metrics and drop http_queue_guard on first token
    let mut http_queue_guard = Some(http_queue_guard);
    let stream = stream.inspect(move |response| {
        process_response_and_observe_metrics(
            response,
            &mut response_collector,
            &mut http_queue_guard,
        );
    });

    // Fold the (single-response) stream into one classification response.
    let response = NvCreateClassifyResponse::from_annotated_stream(stream)
        .await
        .map_err(|e| {
            let err_response = ErrorMessage::from_anyhow(
                anyhow::Error::new(e),
                "Failed to fold classification stream",
            );
            inflight.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    inflight.mark_ok();
    Ok(Json(response).into_response())
}

fn pooling_or_classify_bad_request(message: String) -> ErrorResponse {
    let code = StatusCode::BAD_REQUEST;
    (
        code,
        Json(ErrorMessage {
            message,
            error_type: map_error_code_to_error_type(code),
            code: code.as_u16(),
            details: None,
            metric_error_type: None,
        }),
    )
}

fn validate_pooling_cache_salt(cache_salt: Option<&str>) -> Result<(), ErrorResponse> {
    if cache_salt == Some("") {
        return Err(pooling_or_classify_bad_request(
            "Parameter 'cache_salt' must be a non-empty string if provided.".to_string(),
        ));
    }
    Ok(())
}

#[derive(Serialize)]
struct PoolingBinaryMetadataItem {
    index: u32,
    embed_dtype: &'static str,
    endianness: &'static str,
    start: usize,
    end: usize,
    shape: Vec<u64>,
}

#[derive(Serialize)]
struct PoolingBinaryUsage {
    prompt_tokens: u32,
    total_tokens: u32,
}

#[derive(Serialize)]
struct PoolingBinaryMetadata {
    id: String,
    created: u64,
    model: String,
    data: Vec<PoolingBinaryMetadataItem>,
    usage: PoolingBinaryUsage,
}

fn build_pooling_binary_response(
    response: NvCreatePoolingResponse,
    include_metadata: bool,
    embed_dtype: PoolingEmbedDType,
    endianness: PoolingEndianness,
) -> anyhow::Result<Response> {
    let NvCreatePoolingResponse {
        id,
        created,
        model,
        data,
        usage,
        ..
    } = response;

    let mut chunks = Vec::with_capacity(data.len());
    let mut metadata_items = Vec::with_capacity(if include_metadata { data.len() } else { 0 });
    let mut offset = 0usize;

    for item in data {
        let encoded = match item.data {
            PoolingOutput::Base64(encoded) => encoded,
            _ => anyhow::bail!(
                "binary pooling output at index {} was not base64 encoded",
                item.index
            ),
        };
        let bytes = base64::engine::general_purpose::STANDARD
            .decode(encoded)
            .map_err(|e| {
                anyhow::anyhow!(
                    "invalid base64 in binary pooling output at index {}: {e}",
                    item.index
                )
            })?;
        let end = offset.checked_add(bytes.len()).ok_or_else(|| {
            anyhow::anyhow!(
                "binary pooling response size overflow at index {}",
                item.index
            )
        })?;

        if include_metadata {
            let shape = item.shape.ok_or_else(|| {
                anyhow::anyhow!(
                    "binary pooling output at index {} is missing its tensor shape",
                    item.index
                )
            })?;
            let expected_len =
                shape
                    .iter()
                    .try_fold(embed_dtype.byte_width(), |size, &dimension| {
                        let dimension = usize::try_from(dimension).map_err(|_| {
                            anyhow::anyhow!(
                                "binary pooling tensor dimension overflow at index {}",
                                item.index
                            )
                        })?;
                        size.checked_mul(dimension).ok_or_else(|| {
                            anyhow::anyhow!(
                                "binary pooling tensor size overflow at index {}",
                                item.index
                            )
                        })
                    })?;
            anyhow::ensure!(
                bytes.len() == expected_len,
                "binary pooling output at index {} has {} bytes, but shape {:?} with dtype {} requires {}",
                item.index,
                bytes.len(),
                shape,
                embed_dtype.as_str(),
                expected_len
            );
            metadata_items.push(PoolingBinaryMetadataItem {
                index: item.index,
                embed_dtype: embed_dtype.as_str(),
                endianness: endianness.as_str(),
                start: offset,
                end,
                shape,
            });
        }

        chunks.push(Bytes::from(bytes));
        offset = end;
    }

    let metadata = if include_metadata {
        Some(serde_json::to_string(&PoolingBinaryMetadata {
            id,
            created,
            model,
            data: metadata_items,
            usage: PoolingBinaryUsage {
                prompt_tokens: usage.prompt_tokens,
                total_tokens: usage.total_tokens,
            },
        })?)
    } else {
        None
    };

    let body = Body::from_stream(stream::iter(
        chunks
            .into_iter()
            .map(Ok::<Bytes, std::convert::Infallible>),
    ));
    let mut builder =
        Response::builder().header(axum::http::header::CONTENT_TYPE, "application/octet-stream");
    if let Some(metadata) = metadata {
        builder = builder.header("metadata", metadata);
    }
    Ok(builder.body(body)?)
}

#[tracing::instrument(skip_all)]
async fn pooling(
    State(state): State<Arc<service_v2::State>>,
    headers: HeaderMap,
    Json(mut request): Json<NvCreatePoolingRequest>,
) -> Result<Response, ErrorResponse> {
    // return a 503 if the service or model is not ready
    check_ready(&state)?;
    check_model_serving_ready(&state, &request.model)?;

    if !state.nvext_enabled() {
        warn_nvext_disabled(
            "pooling",
            request
                .nvext
                .as_ref()
                .is_some_and(|nvext| nvext.annotations.is_some()),
        );
        request.nvext = None;
    }
    let response_encoding = request.encoding_format;
    let response_dtype = request.embed_dtype.unwrap_or_default();
    let response_endianness = request.endianness.unwrap_or_default();

    // Resolve alias → primary served name before wrapping the request, so
    // engine routing, metrics, and the response model all use the canonical
    // primary (mirrors `embeddings` / `completions_single`).
    let canonical = state.manager().resolve_canonical_name(&request.model);
    if canonical != request.model {
        request.model = canonical;
    }
    let request_id = get_or_create_request_id(&headers);
    let request = context_from_headers(request, request_id, &headers)?;
    let request_id = request.id().to_string();

    // Pooling, like embeddings, is a single (non-streaming) response.
    let streaming = false;

    let model = &request.model;
    let metric_model = state.manager().metric_model_for(model).to_string();

    // Create inflight_guard early to ensure all errors (including validation)
    // are counted. Request validation runs after this point so a rejected
    // request still lands in `requests_total` with error_type=validation
    // (mirrors `chat_completions`).
    let mut inflight = state.metrics_clone().create_inflight_guard(
        &metric_model,
        Endpoint::Pooling,
        streaming,
        &request_id,
    );

    // Marked as `Validation` explicitly rather than through
    // `extract_error_type_from_response`: that helper infers the type from the
    // message, and only a `VALIDATION_PREFIX`-prefixed 400 maps to
    // `Validation` (anything else falls back to `Internal`). These messages
    // stay verbatim vLLM-compatible, so the prefix is not an option here.
    if let Err(err_response) = validate_pooling_cache_salt(request.cache_salt.as_deref()) {
        inflight.mark_error(ErrorType::Validation);
        return Err(err_response);
    }

    // vLLM currently rejects dimensionality reduction on `/pooling`.
    if request.dimensions.is_some() {
        inflight.mark_error(ErrorType::Validation);
        return Err(pooling_or_classify_bad_request(
            "dimensions is currently not supported".to_string(),
        ));
    }

    // Create http_queue_guard early - tracks time waiting to be processed
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    let engine = state.manager().get_pooling_engine(model).map_err(|e| {
        let err_response = ErrorMessage::from_model_error(&e);
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    let mut response_collector = state
        .metrics_clone()
        .create_response_collector(&metric_model);
    let model_name = model.to_string();

    // issue the generate call on the engine
    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model_name, super::metrics::Endpoint::Pooling);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate pooling output");
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    // Process stream to collect metrics and drop http_queue_guard on first token
    let mut http_queue_guard = Some(http_queue_guard);
    let stream = stream.inspect(move |response| {
        process_response_and_observe_metrics(
            response,
            &mut response_collector,
            &mut http_queue_guard,
        );
    });

    // Fold the (single-response) stream into one pooling response.
    let response = NvCreatePoolingResponse::from_annotated_stream(stream)
        .await
        .map_err(|e| {
            let err_response =
                ErrorMessage::from_anyhow(anyhow::Error::new(e), "Failed to fold pooling stream");
            inflight.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    let response = match response_encoding {
        PoolingEncodingFormat::Float | PoolingEncodingFormat::Base64 => {
            Json(response).into_response()
        }
        PoolingEncodingFormat::Bytes | PoolingEncodingFormat::BytesOnly => {
            build_pooling_binary_response(
                response,
                response_encoding == PoolingEncodingFormat::Bytes,
                response_dtype,
                response_endianness,
            )
            .map_err(|e| {
                let err_response =
                    ErrorMessage::from_anyhow(e, "Failed to build pooling binary response");
                inflight.mark_error(extract_error_type_from_response(&err_response));
                err_response
            })?
        }
    };

    inflight.mark_ok();
    Ok(response)
}

async fn handler_chat_completions(
    State((state, template)): State<(Arc<service_v2::State>, Option<RequestTemplate>)>,
    headers: HeaderMap,
    body: Body,
) -> Result<Response, ErrorResponse> {
    let body = read_json_request_body(&headers, body).await?;
    let mut request: NvCreateChatCompletionRequest = parse_json_request("chat completions", &body)?;
    if *FORCE_INCLUDE_USAGE && request.inner.stream.unwrap_or(false) {
        delta_common::force_include_usage(&mut request.inner.stream_options);
    }

    // return a 503 if the service is not ready (process-level + per-model
    // serving readiness). An aggregated request to a decode-only namespace
    // would otherwise hang/crash on the decode worker. Resolve the templated
    // model first so empty/missing `model` fields don't bypass the gate.
    check_ready(&state)?;
    let resolved_model = resolve_request_model(&request.inner.model, template.as_ref());
    if !resolved_model.is_empty() {
        check_model_serving_ready(&state, resolved_model)?;
    }

    if !state.nvext_enabled() {
        warn_nvext_disabled(
            "chat_completions",
            request
                .nvext
                .as_ref()
                .is_some_and(CommonNvExt::has_non_cache_salt_fields)
                || has_non_cache_salt_routing_headers(&headers),
        );
    }
    request.nvext =
        apply_frontend_nvext_policy(request.nvext.take(), &headers, state.nvext_enabled());

    // create the context for the request
    let request_id = get_or_create_request_id(&headers);
    let streaming = request.inner.stream.unwrap_or(false);
    let resolved_model = resolve_request_model(&request.inner.model, template.as_ref());
    // Canonicalize alias → primary for the metric label.
    let canonical_model = state.manager().resolve_canonical_name(resolved_model);
    let cancellation_labels = CancellationLabels {
        model: state
            .manager()
            .metric_model_for(&canonical_model)
            .to_string(),
        endpoint: Endpoint::ChatCompletions.to_string(),
        request_type: if streaming { "stream" } else { "unary" }.to_string(),
    };
    let mut request =
        context_from_headers_with_input_trigger(request, request_id, &headers, |request| {
            Some(classify_chat_request(request))
        })?;
    if let Some(captured) = crate::request_trace::payload::capture_http_headers(&headers) {
        request.insert(
            crate::request_trace::payload::HTTP_HEADERS_CONTEXT_KEY,
            captured,
        );
    }
    let context = request.context();

    // create the connection handles
    let (mut connection_handle, stream_handle) = create_connection_monitor(
        context.clone(),
        Some(state.metrics_clone()),
        cancellation_labels,
    )
    .await;

    let response =
        tokio::spawn(chat_completions(state, template, request, stream_handle).in_current_span())
            .await
            .map_err(|e| {
                ErrorMessage::internal_server_error_with_details(
                    "Failed to await chat completions task",
                    format!("{e:?}"),
                )
            })?;

    // if we got here, then we will return a response and the potentially long running task has completed successfully
    // without need to be cancelled.
    connection_handle.disarm();

    response
}

fn parse_json_request<T>(endpoint: &'static str, body: &[u8]) -> Result<T, ErrorResponse>
where
    T: DeserializeOwned,
{
    match serde_json::from_slice(body) {
        Ok(request) => Ok(request),
        Err(original_error) => {
            if let Some(escaped_body) = escape_json_string_control_chars(body) {
                match serde_json::from_slice(&escaped_body) {
                    Ok(request) => {
                        tracing::warn!(
                            endpoint,
                            "Accepted request after escaping unescaped control characters in JSON strings"
                        );
                        Ok(request)
                    }
                    Err(_) => parse_json_request_lossy(endpoint, body)
                        .map_err(|_| json_deserialize_error(original_error)),
                }
            } else {
                parse_json_request_lossy(endpoint, body)
                    .map_err(|_| json_deserialize_error(original_error))
            }
        }
    }
}

fn parse_json_request_lossy<T>(endpoint: &'static str, body: &[u8]) -> Result<T, serde_json::Error>
where
    T: DeserializeOwned,
{
    let lossy_body = String::from_utf8_lossy(body);
    if lossy_body.as_bytes() == body {
        return serde_json::from_slice(body);
    }

    let escaped_body = escape_json_string_control_chars(lossy_body.as_bytes())
        .unwrap_or_else(|| lossy_body.into_owned().into_bytes());
    let request = serde_json::from_slice(&escaped_body)?;
    tracing::warn!(
        endpoint,
        "Accepted request after replacing invalid UTF-8 and escaping unescaped control characters in JSON strings"
    );
    Ok(request)
}

fn json_deserialize_error(error: serde_json::Error) -> ErrorResponse {
    let code = StatusCode::BAD_REQUEST;
    (
        code,
        Json(ErrorMessage {
            message: format!("Failed to deserialize the JSON body into the target type: {error}"),
            error_type: map_error_code_to_error_type(code),
            code: code.as_u16(),
            details: None,
            metric_error_type: None,
        }),
    )
}

fn ensure_json_content_type(headers: &HeaderMap) -> Result<(), ErrorResponse> {
    let Some(content_type) = headers
        .get(axum::http::header::CONTENT_TYPE)
        .and_then(|value| value.to_str().ok())
    else {
        return Err(unsupported_media_type_error());
    };

    if is_json_content_type(content_type) {
        Ok(())
    } else {
        Err(unsupported_media_type_error())
    }
}

fn unsupported_media_type_error() -> ErrorResponse {
    let code = StatusCode::UNSUPPORTED_MEDIA_TYPE;
    (
        code,
        Json(ErrorMessage {
            message: "Expected request with Content-Type application/json".to_string(),
            error_type: map_error_code_to_error_type(code),
            code: code.as_u16(),
            details: None,
            metric_error_type: None,
        }),
    )
}

/// Returns the standard error response for a request body that exceeds the
/// configured size limit.
fn payload_too_large_error() -> ErrorResponse {
    let code = StatusCode::PAYLOAD_TOO_LARGE;
    (
        code,
        Json(ErrorMessage {
            message: format!(
                "Request body exceeds the limit of {} MB set by {}",
                get_body_limit() / (1024 * 1024),
                env_llm::DYN_HTTP_BODY_LIMIT_MB
            ),
            error_type: map_error_code_to_error_type(code),
            code: code.as_u16(),
            details: None,
            metric_error_type: None,
        }),
    )
}

/// Returns the standard error response when the request body cannot be read.
fn failed_to_read_request_body_error() -> ErrorResponse {
    let code = StatusCode::BAD_REQUEST;
    (
        code,
        Json(ErrorMessage {
            message: "Failed to read request body".to_string(),
            error_type: map_error_code_to_error_type(code),
            code: code.as_u16(),
            details: None,
            metric_error_type: None,
        }),
    )
}

/// Reads and buffers a JSON request body.
///
/// Validates the `Content-Type` before reading the body and limits buffering to [`get_body_limit`].
async fn read_json_request_body(headers: &HeaderMap, body: Body) -> Result<Bytes, ErrorResponse> {
    ensure_json_content_type(headers)?;
    axum::body::to_bytes(body, get_body_limit())
        .await
        .map_err(|error| {
            // `to_bytes` wraps an oversized-body failure in its error source
            // rather than returning `LengthLimitError` directly.
            if std::error::Error::source(&error)
                .is_some_and(|source| source.is::<LengthLimitError>())
            {
                payload_too_large_error()
            } else {
                failed_to_read_request_body_error()
            }
        })
}

fn is_json_content_type(content_type: &str) -> bool {
    let media_type = content_type.split(';').next().unwrap_or_default().trim();
    let Some((media_type, subtype)) = media_type.split_once('/') else {
        return false;
    };

    media_type.eq_ignore_ascii_case("application")
        && (subtype.eq_ignore_ascii_case("json")
            || subtype
                .to_ascii_lowercase()
                .rsplit_once('+')
                .is_some_and(|(_, suffix)| suffix == "json"))
}

fn escape_json_string_control_chars(body: &[u8]) -> Option<Vec<u8>> {
    let mut out = Vec::with_capacity(body.len());
    let mut in_string = false;
    let mut escaped = false;
    let mut changed = false;

    for &byte in body {
        if in_string && byte <= 0x1f {
            const HEX: &[u8; 16] = b"0123456789abcdef";
            if escaped {
                out.extend_from_slice(b"\\\\u00");
                escaped = false;
            } else {
                out.extend_from_slice(b"\\u00");
            }
            out.push(HEX[(byte >> 4) as usize]);
            out.push(HEX[(byte & 0x0f) as usize]);
            changed = true;
            continue;
        }

        out.push(byte);

        if escaped {
            escaped = false;
        } else if in_string && byte == b'\\' {
            escaped = true;
        } else if byte == b'"' {
            in_string = !in_string;
        }
    }

    changed.then_some(out)
}

/// A backend error extracted from an event, ready for `backend_error_response`.
struct BackendErrorInfo {
    message: String,
    status: StatusCode,
    /// Classification already established from the error chain, when the
    /// status alone is not enough to recover it. `None` means "derive it from
    /// `status`" — the ordinary case for a status the worker supplied.
    sanitized: Option<SanitizedError>,
}

impl BackendErrorInfo {
    /// The common case: nothing known beyond what the worker reported.
    fn from_status(message: String, status: StatusCode) -> Self {
        Self {
            message,
            status,
            sanitized: None,
        }
    }
}

/// The `{"message": ..., "code": ...}` envelope `py_err_to_dynamo` emits for an
/// HTTP-like Python exception (see `lib/bindings/python/rust/backend.rs`).
///
/// A worker's error message is therefore not always prose, and the raw envelope
/// must never reach a client. Both the in-stream path
/// ([`extract_backend_error_if_present`]) and the pre-stream path
/// ([`ErrorMessage::from_anyhow`]) unwrap it, so the *body* is the same either
/// way. They deliberately differ on the *status*: the in-stream path honours
/// any code in the envelope, while the pre-stream path honours only a client
/// error, because it is reached solely through the `InvalidArgument` arm and a
/// 5xx there would skip the sanitizing the other arms apply.
#[derive(serde::Deserialize)]
struct ErrorPayload {
    message: Option<String>,
    code: Option<u16>,
}

/// Checks if an Annotated event represents a backend error and extracts error information.
/// Returns Some(info) if it's an error, None otherwise.
fn extract_backend_error_if_present<T: serde::Serialize>(
    event: &Annotated<T>,
) -> Option<BackendErrorInfo> {
    // Check if event type is "error" (from postprocessor when FinishReason::Error is encountered)
    if let Some(event_type) = &event.event
        && event_type == "error"
    {
        use dynamo_runtime::error::{BackendError, ErrorType};

        // Classify only this event's error, not its causes. An inner invalid
        // argument must not override an outer unavailable or internal error.
        let invalid_argument = event.error.as_ref().filter(|error| {
            matches!(
                error.error_type(),
                ErrorType::InvalidArgument | ErrorType::Backend(BackendError::InvalidArgument)
            )
        });

        // Extract error string: prefer DynamoError field, fallback to legacy comment.
        // Use message() instead of to_string() for DynamoError to avoid prefixing
        // the ErrorType (e.g., "Unknown: {...}"), which would break JSON parsing.
        let error_str = if let Some(ref dynamo_err) = event.error {
            let mut parts = Vec::new();
            let mut current: Option<&dyn std::error::Error> = Some(dynamo_err);
            while let Some(e) = current {
                if let Some(de) = e.downcast_ref::<dynamo_runtime::error::DynamoError>() {
                    parts.push(de.message().to_string());
                } else {
                    parts.push(e.to_string());
                }
                current = e.source();
            }
            parts.join(", ")
        } else {
            event
                .comment
                .as_ref()
                .map(|c| c.join(", "))
                .unwrap_or_else(|| "Unknown error".to_string())
        };

        // Capacity rejection is not a backend fault. Workers report it with their
        // own status (503), which a client cannot tell from a real outage, so the
        // error chain wins over the payload code — admission-path and worker-path
        // rejections then surface identically. See DYN_HTTP_OVERLOAD_STATUS_CODE.
        let overloaded = event
            .error
            .as_ref()
            .is_some_and(|error| super::metrics::request_was_rejected(error));

        // Parse the status-bearing node's own message. The diagnostic string
        // above includes its causes and therefore is not necessarily JSON.
        let status_message = event
            .error
            .as_ref()
            .map(|error| error.message())
            .unwrap_or(&error_str);
        if let Ok(error_payload) = serde_json::from_str::<ErrorPayload>(status_message) {
            // Preserve explicit HTTP-like statuses (for example 415); Python
            // 4xx exceptions share the Backend(InvalidArgument) category.
            let code = if overloaded {
                overload_status_code()
            } else {
                match error_payload.code {
                    Some(code) => {
                        StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR)
                    }
                    None if invalid_argument.is_some() => StatusCode::BAD_REQUEST,
                    None => StatusCode::INTERNAL_SERVER_ERROR,
                }
            };
            let message = error_payload
                .message
                .unwrap_or_else(|| status_message.to_string());
            return Some(BackendErrorInfo {
                message,
                status: code,
                sanitized: overloaded.then_some(SanitizedError::Overloaded),
            });
        }

        if let Some(invalid_argument) = invalid_argument {
            return Some(BackendErrorInfo::from_status(
                invalid_argument.message().to_string(),
                StatusCode::BAD_REQUEST,
            ));
        }

        if overloaded {
            return Some(BackendErrorInfo {
                message: error_str,
                status: overload_status_code(),
                sanitized: Some(SanitizedError::Overloaded),
            });
        }

        return Some(BackendErrorInfo::from_status(
            error_str,
            StatusCode::INTERNAL_SERVER_ERROR,
        ));
    }

    // Check if the data payload itself contains an error structure with code >= 400
    if let Some(data) = &event.data
        && let Ok(json_value) = serde_json::to_value(data)
        && let Ok(error_payload) = serde_json::from_value::<ErrorPayload>(json_value.clone())
        && let Some(code_num) = error_payload.code
        && code_num >= 400
    {
        let code = StatusCode::from_u16(code_num).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
        let message = error_payload
            .message
            .unwrap_or_else(|| json_value.to_string());
        return Some(BackendErrorInfo::from_status(message, code));
    }

    // Check if comment contains error information (without event: error)
    if let Some(comments) = &event.comment
        && !comments.is_empty()
    {
        let comment_str = comments.join(", ");

        // Try to parse comment as error JSON with code >= 400
        if let Ok(error_payload) = serde_json::from_str::<ErrorPayload>(&comment_str)
            && let Some(code_num) = error_payload.code
            && code_num >= 400
        {
            let code = StatusCode::from_u16(code_num).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
            let message = error_payload.message.unwrap_or(comment_str);
            return Some(BackendErrorInfo::from_status(message, code));
        }

        // Comments present with no data AND no event type indicates error
        // (events with event types like "request_id" or "event.dynamo.test.sentinel" are annotations)
        if event.data.is_none() && event.event.is_none() {
            return Some(BackendErrorInfo::from_status(
                comment_str,
                StatusCode::INTERNAL_SERVER_ERROR,
            ));
        }
    }

    None
}

/// Returns true for events that only carry an annotation tag (e.g. the
/// `request_id` frame prepended to every stream): no data, no error, and
/// an `event` field that is *not* the `"error"` marker. Annotations may
/// still carry a serialized value in `comment` (that is how
/// `Annotated::from_annotation` builds them), so the comment field is
/// not part of the check. These frames are stepped over by
/// `check_for_backend_error` so an immediate backend error in the *next*
/// slot is still caught instead of slipping through to the fold/parse
/// path.
fn is_annotation_frame<T>(e: &Annotated<T>) -> bool {
    e.data.is_none()
        && e.error.is_none()
        && matches!(e.event.as_deref(), Some(tag) if tag != "error")
}

/// Cap on how many leading annotation frames `check_for_backend_error`
/// will buffer before giving up the inspection. A pathological backend
/// (or attacker who can influence the engine output) that emits only
/// annotation frames must not be able to pin unbounded memory per
/// request. The handful of real annotations a frontend prepends
/// (currently just `request_id`) fits well under this cap.
const MAX_LEADING_ANNOTATIONS: usize = 16;

/// Inspect the first non-annotation event in the stream for a backend error.
///
/// `timeout = None` — await stream events indefinitely (non-streaming preflight).
/// `timeout = Some(dur)` — race against a single deadline captured at function
/// entry (streaming pre-commit peek). If the deadline elapses before a
/// non-annotation event arrives, return the buffered annotations chained with
/// the remaining stream so downstream sees the original ordering.
///
/// Returns `Err(ErrorResponse)` if the first non-annotation event is a backend
/// error, `Ok(stream)` otherwise.
pub(super) async fn check_for_backend_error<T>(
    stream: impl futures::Stream<Item = Annotated<T>> + Send + 'static,
    timeout: Option<std::time::Duration>,
) -> Result<std::pin::Pin<Box<dyn futures::Stream<Item = Annotated<T>> + Send>>, ErrorResponse>
where
    T: serde::Serialize + Send + 'static,
{
    use futures::stream::StreamExt;

    let mut stream = Box::pin(stream);
    // Single deadline captured at entry so the peek window is bounded in total,
    // not per-iteration.
    let deadline = timeout.map(|t| tokio::time::Instant::now() + t);
    let mut buffered: Vec<Annotated<T>> = Vec::new();

    loop {
        let next = match deadline {
            Some(d) => tokio::select! {
                item = stream.next() => item,
                _ = tokio::time::sleep_until(d) => {
                    return Ok(Box::pin(futures::stream::iter(buffered).chain(stream)));
                }
            },
            None => stream.next().await,
        };

        let Some(event) = next else {
            // Backend closed before yielding any non-annotation event; replay
            // buffered annotations so downstream sees them.
            return Ok(Box::pin(futures::stream::iter(buffered)));
        };

        if is_annotation_frame(&event) && buffered.len() < MAX_LEADING_ANNOTATIONS {
            buffered.push(event);
            continue;
        }

        if let Some(backend_error) = extract_backend_error_if_present(&event) {
            return Err(backend_error_response(backend_error));
        }

        // First non-annotation, non-error event — hand back for downstream
        // consumption with original ordering preserved.
        buffered.push(event);
        return Ok(Box::pin(futures::stream::iter(buffered).chain(stream)));
    }
}

/// Convert a `BackendErrorInfo` from `extract_backend_error_if_present` into the
/// wire `ErrorResponse`. Shared between the non-streaming preflight
/// (`check_for_backend_error`) and the streaming preflight so both paths speak
/// the same sanitization + status contract to the client.
///
/// The streaming counterpart of [`ErrorMessage::from_http_error`], and it must
/// triage identically. Both once called [`SanitizedError::for_backend_status`]
/// directly, which preserves every 5xx on the wire — including ones that
/// should have been coerced to 500 because they do not keep retry semantics.
/// Only the unary path moved to [`BackendStatusAction`], so the same backend
/// failure answered 502 when it arrived mid-stream and 500 when it arrived as
/// an `HttpError`, and a client could not tell which it would get.
///
/// A classification carried on `backend_error.sanitized` wins over one derived
/// from the status: the status alone cannot distinguish a capacity rejection
/// from an outage once `DYN_HTTP_OVERLOAD_STATUS_CODE` is set outside the 5xx
/// range.
fn backend_error_response(backend_error: BackendErrorInfo) -> ErrorResponse {
    let BackendErrorInfo {
        message,
        status,
        sanitized,
    } = backend_error;
    let action = match sanitized {
        Some(variant) => BackendStatusAction::Sanitize(variant),
        None => BackendStatusAction::triage(status),
    };
    match action {
        BackendStatusAction::Sanitize(variant) => {
            ErrorMessage::sanitized_with_details(variant, message)
        }
        BackendStatusAction::CoerceToInternal(asserted) => {
            ErrorMessage::coerced_backend_error(asserted, message)
        }
        // 4xx (non-499): protocol contract — forward backend message as-is.
        BackendStatusAction::ForwardClientError => (
            status,
            Json(ErrorMessage {
                message,
                error_type: map_error_code_to_error_type(status),
                code: status.as_u16(),
                details: None,
                metric_error_type: None,
            }),
        ),
    }
}

/// Read the pre-commit peek window from the environment.
///
/// `Some(dur)` — poll for that duration before committing SSE.
/// `None` — the peek is disabled entirely (default; matches pre-fix behavior
/// where all backend errors surface as SSE frames post-HTTP-200).
///
/// Read live per streaming request. Reading `std::env::var` is a hashmap
/// lookup — sub-microsecond, negligible next to the peek window
/// itself. Live reads make the value tunable at test time without a
/// process restart.
// FIXME: unify env-var initialization with the rest of `env_llm::*` once that
// module gets a standard reader.
fn pre_commit_error_peek_timeout() -> Option<std::time::Duration> {
    match std::env::var(env_llm::DYN_HTTP_PRE_COMMIT_ERROR_PEEK_MS)
        .ok()
        .and_then(|s| s.parse::<u64>().ok())
    {
        Some(0) | None => None,
        Some(ms) => Some(std::time::Duration::from_millis(ms)),
    }
}

#[derive(Serialize)]
struct ToolCallDispatchPayload<'a> {
    choice_index: u32,
    tool_call: &'a ChatCompletionMessageToolCallChunk,
}

#[derive(Serialize)]
struct ReasoningDispatchPayload<'a> {
    index: u32,
    reasoning_content: &'a str,
}

/// Serialize `payload` and append it as an SSE event with the given name.
fn push_dispatch_event(
    event_name: &str,
    payload: &impl serde::Serialize,
    out: &mut Vec<Result<Event, axum::Error>>,
) {
    match serde_json::to_string(payload) {
        Ok(json) => out.push(Ok(Event::default().event(event_name).data(json))),
        Err(e) => {
            tracing::warn!("streaming_{event_name}: failed to serialize: {e}");
        }
    }
}

/// Empty stream chunk produced by multi-byte token assembly (e.g. emoji).
fn is_empty_stream_response(resp: &NvCreateChatCompletionStreamResponse) -> bool {
    if resp.nvext.is_some() {
        return false;
    }
    resp.inner.usage.is_none()
        && resp.inner.choices.iter().all(|c| {
            let ChatCompletionStreamResponseDelta {
                content,
                function_call,
                tool_calls,
                role,
                refusal,
                reasoning_content,
            } = &c.delta;
            // `Text("")` happens during multi-byte UTF-8 token assembly;
            // `Parts(vec![])` is a structurally empty multimodal payload.
            let content_empty = match content {
                None => true,
                Some(ChatCompletionMessageContent::Text(t)) => t.is_empty(),
                Some(ChatCompletionMessageContent::Parts(p)) => p.is_empty(),
            };
            c.finish_reason.is_none()
                && c.logprobs.is_none()
                && content_empty
                && function_call.is_none()
                && tool_calls.is_none()
                && role.is_none()
                && refusal.is_none()
                && reasoning_content.is_none()
        })
}

/// Preserve the first role delta for each choice and remove parser-generated repeats.
fn deduplicate_stream_roles(
    resp: &mut NvCreateChatCompletionStreamResponse,
    emitted_roles: &mut HashSet<u32>,
) {
    for choice in &mut resp.inner.choices {
        if choice.delta.role.is_some() && !emitted_roles.insert(choice.index) {
            choice.delta.role = None;
        }
    }
}

/// Completions variant of [`is_empty_stream_response`].
fn is_empty_completion_stream_response(resp: &NvCreateCompletionResponse) -> bool {
    if resp.nvext.is_some() {
        return false;
    }
    resp.inner.usage.is_none()
        && resp.inner.choices.iter().all(|c| {
            let Choice {
                text,
                index: _,
                logprobs,
                finish_reason,
            } = c;
            text.is_empty() && finish_reason.is_none() && logprobs.is_none()
        })
}

/// Emits early `event: tool_call_dispatch` SSE events for any complete tool calls found in a
/// streaming response chunk, when `DYN_ENABLE_STREAMING_TOOL_DISPATCH` is enabled.
///
/// Dynamo backends emit each tool call as a single complete chunk (id + name + arguments
/// all present), so we can dispatch immediately upon seeing the chunk rather than waiting
/// for `finish_reason="tool_calls"` to arrive. Each event payload includes `choice_index`
/// for correct disambiguation when `n > 1`.
///
/// Dedup is keyed by `(choice_index, tool_call_id)`, not by id alone: with `n > 1` the
/// backend may reuse the same tool call id across choices, and keying on the id alone
/// would silently drop every choice after the first.
fn streaming_tool_dispatch_events(
    response: &crate::types::Annotated<NvCreateChatCompletionStreamResponse>,
    dispatched_ids: &mut HashSet<(u32, String)>,
    out: &mut Vec<Result<Event, axum::Error>>,
) {
    let Some(data) = &response.data else {
        return;
    };

    for choice in &data.inner.choices {
        let Some(tool_calls) = &choice.delta.tool_calls else {
            continue;
        };
        for chunk in tool_calls {
            // Only dispatch when the tool call is fully formed (id + name + arguments)
            let has_name_and_args = chunk
                .function
                .as_ref()
                .is_some_and(|f| f.name.is_some() && f.arguments.is_some());

            if let (true, Some(id)) = (has_name_and_args, &chunk.id) {
                // Skip already-dispatched tool calls (dedup guard, matches
                // the stopped/done flags in Anthropic/Responses converters).
                // Scoped per choice so repeated ids across choices still dispatch.
                if !dispatched_ids.insert((choice.index, id.clone())) {
                    continue;
                }
                let payload = ToolCallDispatchPayload {
                    choice_index: choice.index,
                    tool_call: chunk,
                };
                push_dispatch_event("tool_call_dispatch", &payload, out);
            }
        }
    }
}

/// Accumulates reasoning tokens and emits a single `event: reasoning_dispatch` SSE event
/// when the complete reasoning block has been decoded (i.e. when `reasoning_content`
/// transitions from `Some(token)` to `None`), matching the UX of `tool_call_dispatch`.
///
/// The buffer is maintained across chunks by the caller (captured in the flat_map closure).
/// Flushing also occurs when `finish_reason` is set, to handle max_tokens during reasoning.
fn accumulate_reasoning_dispatch(
    response: &crate::types::Annotated<NvCreateChatCompletionStreamResponse>,
    buffers: &mut HashMap<u32, String>,
    out: &mut Vec<Result<Event, axum::Error>>,
) {
    let Some(data) = &response.data else {
        return;
    };

    for choice in &data.inner.choices {
        let buffer = buffers.entry(choice.index).or_default();
        let has_reasoning = choice
            .delta
            .reasoning_content
            .as_ref()
            .is_some_and(|r| !r.is_empty());

        if has_reasoning {
            buffer.push_str(choice.delta.reasoning_content.as_ref().unwrap());
        }

        // Emit when reasoning transitions to None OR when the stream ends (finish_reason).
        if !buffer.is_empty() && (!has_reasoning || choice.finish_reason.is_some()) {
            let payload = ReasoningDispatchPayload {
                index: choice.index,
                reasoning_content: buffer.as_str(),
            };
            push_dispatch_event("reasoning_dispatch", &payload, out);
            buffer.clear();
        }
    }
}

fn apply_chat_completions_request_template(
    request: &mut dynamo_protocols::types::CreateChatCompletionRequest,
    template: Option<&RequestTemplate>,
) {
    if let Some(template) = template {
        if request.model.is_empty() {
            request.model = template.model.clone();
        }
        if request.temperature.is_none() {
            request.temperature = Some(template.temperature);
        }
        if request.max_completion_tokens.unwrap_or(0) == 0 {
            request.max_completion_tokens = Some(template.max_completion_tokens);
        }
    }
}

/// OpenAI Chat Completions Request Handler
///
/// This method will handle the incoming request for the /v1/chat/completions endpoint. The endpoint is a "source"
/// for an [`super::OpenAIChatCompletionsStreamingEngine`] and will return a stream of responses which will be
/// forward to the client.
///
/// Note: For all requests, streaming or non-streaming, we always call the engine with streaming enabled. For
/// non-streaming requests, we will fold the stream into a single response as part of this handler.
async fn chat_completions(
    state: Arc<service_v2::State>,
    template: Option<RequestTemplate>,
    mut request: Context<NvCreateChatCompletionRequest>,
    stream_handle: ConnectionHandle,
) -> Result<Response, ErrorResponse> {
    // return a 503 if the service is not ready
    check_ready(&state)?;

    let request_id = request.id().to_string();

    // Determine streaming mode early
    // todo - decide on default
    let streaming = request.inner.stream.unwrap_or(false);

    // Apply template values first to resolve the model before creating metrics guards
    apply_chat_completions_request_template(&mut request.inner, template.as_ref());
    // Capture the resolved model after template application for metrics and engine lookup
    // todo - make the protocols be optional for model name
    // todo - when optional, if none, apply a default
    // todo - determine the proper error code for when a request model is not present
    // Resolve an alias to its primary served name and rewrite the request so
    // engine routing, metrics, and the OpenAI response.model all use the
    // canonical primary (matching vLLM/SGLang). Non-aliases pass through so
    // metric_model_for still applies its unknown-model cardinality guard.
    let canonical = state.manager().resolve_canonical_name(&request.inner.model);
    if canonical != request.inner.model {
        request.inner.model = canonical;
    }
    let model = request.inner.model.clone();
    let metric_model = state.manager().metric_model_for(&model).to_string();

    tracing::trace!("Received chat completions request: {:?}", request.content());

    // Create inflight_guard early to ensure all errors (including validation) are counted
    let mut inflight_guard = state.metrics_clone().create_inflight_guard(
        &metric_model,
        Endpoint::ChatCompletions,
        streaming,
        &request_id,
    );

    if let Err(err_response) = normalize_chat_reasoning_template_args(&mut request) {
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        return Err(err_response);
    }

    // Handle unsupported fields - if Some(resp) is returned by
    // validate_chat_completion_unsupported_fields,
    // then a field was used that is unsupported. We will log an error message
    // and early return a 501 NOT_IMPLEMENTED status code. Otherwise, proceeed.
    if let Err(err_response) = validate_chat_completion_unsupported_fields(&request) {
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        return Err(err_response);
    }

    // Handle required fields like messages shouldn't be empty.
    if let Err(err_response) = validate_chat_completion_required_fields(&request) {
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        return Err(err_response);
    }

    // Validate stream_options is only used when streaming (NVBug 5662680)
    if let Err(err_response) = validate_chat_completion_stream_options(&request) {
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        return Err(err_response);
    }

    // Handle Rest of Validation Errors
    if let Err(err_response) = validate_chat_completion_fields_generic(&request) {
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        return Err(err_response);
    }

    // Create HTTP queue guard after template resolution so labels are correct
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    // Let backend adapters apply their own generation default (e.g. --override-generation-config).
    if request.inner.max_completion_tokens.is_none() {
        request.insert(PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY, true);
    }

    tracing::trace!("Getting chat completions engine for model: {}", model);

    let (engine, parsing_options) = state
        .manager()
        .get_chat_completions_engine_with_parsing(&model)
        .map_err(|e| {
            let err_response = ErrorMessage::from_model_error(&e);
            inflight_guard.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    // Request policy controls whether parser-produced tool calls may be exposed.
    // Assistant response/guided constraints are handled separately during
    // preprocessing and do not revoke an auto request's tool-call permission.
    let parsing_options = apply_request_tool_call_parsing_options(parsing_options, &request)
        .map_err(|e| {
            let err_response = ErrorMessage::from_anyhow(e.into(), "Invalid tool_choice");
            inflight_guard.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    // When parallel_tool_calls is false, limit the response to a single tool call.
    let parsing_options =
        parsing_options.with_parallel_tool_calls(request.inner.parallel_tool_calls);
    let enforce_single_tool_call = request.inner.parallel_tool_calls == Some(false);

    // Any force_nonempty_content=true request: surface reasoning as content when
    // the turn produced none. See `wants_reasoning_as_content_when_empty`.
    let move_reasoning_to_content_when_empty =
        crate::preprocessor::OpenAIPreprocessor::wants_reasoning_as_content_when_empty(
            request.chat_template_args.as_ref(),
        );
    let parsing_options = parsing_options
        .with_move_reasoning_to_content_when_empty(move_reasoning_to_content_when_empty);

    // Computed before `request` moves into `generate`. Only a stream that can
    // withhold every data frame needs forced keep-alive frames.
    let stream_can_defer_all_output =
        request_stream_can_defer_all_output(&parsing_options, request.chat_template_args.as_ref());

    let mut response_collector = state
        .metrics_clone()
        .create_response_collector(&metric_model);

    let annotations = request.annotations();

    // issue the generate call on the engine
    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model, super::metrics::Endpoint::ChatCompletions);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate completions");
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    // capture the context to cancel the stream if the client disconnects
    let ctx = stream.context();

    // prepare any requested annotations
    let annotations = annotations.map_or(Vec::new(), |annotations| {
        annotations
            .iter()
            .filter_map(|annotation| {
                if annotation == ANNOTATION_REQUEST_ID {
                    Annotated::from_annotation(ANNOTATION_REQUEST_ID, &request_id).ok()
                } else {
                    None
                }
            })
            .collect::<Vec<_>>()
    });

    // apply any annotations to the front of the stream
    let stream = stream::iter(annotations).chain(stream);

    // todo - tap the stream and propagate request level metrics
    // note - we might do this as part of the post processing set to make it more generic

    if streaming {
        // Peek the first non-annotation event for a synchronous backend error
        // (e.g. `Backend(InvalidArgument)` from a text-only model receiving
        // image content) before committing HTTP 200, so we can return the
        // typed 4xx that the non-streaming path returns. The peek window is
        // short (`DYN_HTTP_PRE_COMMIT_ERROR_PEEK_MS`) — if no signal arrives,
        // fall through to SSE, and `monitor_for_disconnects` owns the long
        // backend-inactivity timeout from there.
        let stream = match pre_commit_error_peek_timeout() {
            Some(dur) => check_for_backend_error(stream, Some(dur))
                .await
                .map_err(|err_response| {
                    tracing::error!(request_id = %request_id, "Backend error detected: {:?}", err_response);
                    inflight_guard.mark_error(extract_error_type_from_response(&err_response));
                    err_response
                })?,
            // Env var unset → skip peek, commit HTTP 200 immediately (pre-fix
            // behavior). Backend errors will surface as SSE error frames via
            // monitor_for_disconnects.
            None => Box::pin(stream)
                as std::pin::Pin<
                    Box<dyn futures::Stream<Item = _> + Send>,
                >,
        };

        let mut http_queue_guard = Some(http_queue_guard);
        let tool_dispatch_enabled = state.streaming_tool_dispatch_enabled();
        let reasoning_dispatch_enabled = state.streaming_reasoning_dispatch_enabled();
        let reasoning_field = state.reasoning_field();
        let mut reasoning_buffer: HashMap<u32, String> = HashMap::new();
        let mut dispatched_tool_ids: HashSet<(u32, String)> = HashSet::new();
        let mut emitted_roles: HashSet<u32> = HashSet::new();

        // Optionally prepend extra SSE events before each regular chunk:
        //   - `event: tool_call_dispatch`  — complete tool call detected early (tool dispatch)
        //   - `event: reasoning_dispatch`  — complete reasoning block (emitted once)
        let (activity_tx, activity_rx) = tokio::sync::mpsc::unbounded_channel();
        let stream = async_stream::stream! {
            let mut stream = Box::pin(stream);
            let mut events: Vec<Result<Event, axum::Error>> = Vec::with_capacity(4);

            while let Some(mut response) = stream.next().await {
                events.clear();

                // When parallel_tool_calls is false, surface only the first tool call
                // Keep index 0 and drop any higher indexes
                if enforce_single_tool_call
                    && let Some(data) = response.data.as_mut()
                {
                    for choice in data.inner.choices.iter_mut() {
                        if let Some(tool_calls) = choice.delta.tool_calls.as_mut() {
                            tool_calls.retain(|tc| tc.index == 0);
                            if tool_calls.is_empty() {
                                choice.delta.tool_calls = None;
                            }
                        }
                    }
                }

                if let Some(data) = response.data.as_mut() {
                    deduplicate_stream_roles(data, &mut emitted_roles);
                }

                // Drop empty chunks from multi-byte token assembly.
                if response.data.as_ref().is_some_and(is_empty_stream_response) {
                    let _ = activity_tx.send(());
                    // Not forwarded, but the engine still generated these tokens,
                    // so account for them before discarding. Otherwise the
                    // real-time output-token counter undercounts and TTFT is
                    // attributed to the first *renderable* chunk rather than the
                    // first generated one. This already affected multi-byte token
                    // assembly; the Nemotron force_nonempty_content deferral makes
                    // empty chunks common enough to matter.
                    process_chat_response_and_observe_metrics(
                        &response,
                        &mut response_collector,
                        &mut http_queue_guard,
                    );
                    continue;
                }
                if tool_dispatch_enabled {
                    streaming_tool_dispatch_events(
                        &response,
                        &mut dispatched_tool_ids,
                        &mut events,
                    );
                }
                if reasoning_dispatch_enabled {
                    accumulate_reasoning_dispatch(
                        &response,
                        &mut reasoning_buffer,
                        &mut events,
                    );
                }

                // Convert to SSE event (this consumes the response).
                // EventConverter will detect `event: "error"` and convert to SSE error events.
                let sse_result = process_chat_response_using_event_converter_and_observe_metrics(
                    EventConverter::from(response),
                    &mut response_collector,
                    &mut http_queue_guard,
                    reasoning_field,
                );

                // Side-channel events come first, then the regular data event.
                match sse_result {
                    Ok(Some(ev)) => events.push(Ok(ev)),
                    Ok(None) => {}
                    Err(e) => events.push(Err(e)),
                }

                events.reverse();
                while let Some(event) = events.pop() {
                    yield event;
                }
            }
        };
        let keep_alive = state.sse_keep_alive_for_response(stream_can_defer_all_output);
        let stream = monitor_for_disconnects_with_activity(
            stream,
            ctx,
            inflight_guard,
            stream_handle,
            activity_rx,
        );

        let mut sse_stream = Sse::new(stream);
        if let Some(keep_alive) = keep_alive {
            sse_stream = sse_stream.keep_alive(KeepAlive::default().interval(keep_alive));
        }
        Ok(sse_stream.into_response())
    } else {
        // Check first event for backend errors before aggregating (non-streaming only)
        let stream_with_check =
            check_for_backend_error(stream, None)
                .await
                .map_err(|error_response| {
                    tracing::error!(request_id, "Backend error detected: {:?}", error_response);
                    inflight_guard.mark_error(extract_error_type_from_response(&error_response));
                    error_response
                })?;

        let mut http_queue_guard = Some(http_queue_guard);
        let stream = stream_with_check.inspect(move |response| {
            // Calls observe_response() on each token - drops http_queue_guard on first token
            process_chat_response_and_observe_metrics(
                response,
                &mut response_collector,
                &mut http_queue_guard,
            );
        });

        let response =
            NvCreateChatCompletionResponse::from_annotated_stream(stream, parsing_options.clone())
                .await
                .map_err(|e| {
                    tracing::error!(
                        request_id,
                        "Failed to parse chat completion response: {:?}",
                        e
                    );
                    let err_response = ErrorMessage::internal_server_error(
                        "Failed to parse chat completion response",
                    );
                    inflight_guard.mark_error(extract_error_type_from_response(&err_response));
                    err_response
                })?;

        inflight_guard.mark_ok();
        // If the engine context was killed (client disconnect), the response was
        // assembled but never delivered. Override to cancelled.
        if ctx.is_killed() {
            inflight_guard.mark_error(ErrorType::Cancelled);
        }
        Ok(Json(crate::reasoning_field::RoutedReasoning::new(
            response,
            state.reasoning_field(),
        ))
        .into_response())
    }
}

/// Checks for unsupported fields in the request.
/// Returns Some(response) if unsupported fields are present.
#[allow(deprecated)]
pub fn validate_chat_completion_unsupported_fields(
    request: &NvCreateChatCompletionRequest,
) -> Result<(), ErrorResponse> {
    let inner = &request.inner;

    if inner.function_call.is_some() {
        return Err(ErrorMessage::not_implemented_error(
            VALIDATION_PREFIX.to_string()
                + "`function_call` is deprecated. Please migrate to use `tool_choice` instead.",
        ));
    }

    if inner.functions.is_some() {
        return Err(ErrorMessage::not_implemented_error(
            VALIDATION_PREFIX.to_string()
                + "`functions` is deprecated. Please migrate to use `tools` instead.",
        ));
    }

    Ok(())
}

/// Normalizes OpenAI-style reasoning controls before chat completion validation.
fn normalize_chat_reasoning_template_args(
    request: &mut NvCreateChatCompletionRequest,
) -> Result<(), ErrorResponse> {
    request.normalize_reasoning_template_args().map_err(|e| {
        ErrorMessage::from_http_error(HttpError {
            code: 400,
            message: VALIDATION_PREFIX.to_string() + &e.to_string(),
        })
    })
}

fn request_stream_can_defer_all_output(
    parsing_options: &ParsingOptions,
    chat_template_args: Option<&HashMap<String, serde_json::Value>>,
) -> bool {
    crate::preprocessor::OpenAIPreprocessor::stream_can_defer_all_output(
        parsing_options.tool_call_parser.as_deref(),
        parsing_options.reasoning_parser.as_deref(),
        chat_template_args,
    )
}

/// Validates that required fields are present and valid in the chat completion request
pub fn validate_chat_completion_required_fields(
    request: &NvCreateChatCompletionRequest,
) -> Result<(), ErrorResponse> {
    let inner = &request.inner;

    if inner.messages.is_empty() {
        return Err(ErrorMessage::from_http_error(HttpError {
            code: 400,
            message: VALIDATION_PREFIX.to_string()
                + "The 'messages' field cannot be empty. At least one message is required.",
        }));
    }

    Ok(())
}

/// Validates that stream_options is only used when stream=true for chat completions (NVBug 5662680)
pub fn validate_chat_completion_stream_options(
    request: &NvCreateChatCompletionRequest,
) -> Result<(), ErrorResponse> {
    let inner = &request.inner;
    let streaming = inner.stream.unwrap_or(false);
    if !streaming && inner.stream_options.is_some() {
        return Err(ErrorMessage::from_http_error(HttpError {
            code: 400,
            message: VALIDATION_PREFIX.to_string()
                + "The 'stream_options' field is only allowed when 'stream' is set to true.",
        }));
    }
    Ok(())
}

/// Validates a chat completion request and returns an error response if validation fails.
///
/// This function calls the `validate` method implemented for `NvCreateChatCompletionRequest`.
/// If validation fails, it maps the error into an OpenAI-compatible error response.
pub fn validate_chat_completion_fields_generic(
    request: &NvCreateChatCompletionRequest,
) -> Result<(), ErrorResponse> {
    request.validate().map_err(|e| {
        ErrorMessage::from_http_error(HttpError {
            code: 400,
            message: VALIDATION_PREFIX.to_string() + &e.to_string(),
        })
    })
}

/// Validates that stream_options is only used when stream=true for completions (NVBug 5662680)
pub fn validate_completion_stream_options(
    request: &NvCreateCompletionRequest,
) -> Result<(), ErrorResponse> {
    let inner = &request.inner;
    let streaming = inner.stream.unwrap_or(false);
    if !streaming && inner.stream_options.is_some() {
        return Err(ErrorMessage::from_http_error(HttpError {
            code: 400,
            message: VALIDATION_PREFIX.to_string()
                + "The 'stream_options' field is only allowed when 'stream' is set to true.",
        }));
    }
    Ok(())
}

/// Validates a completion request and returns an error response if validation fails.
///
/// This function calls the `validate` method implemented for `NvCreateCompletionRequest`.
/// If validation fails, it maps the error into an OpenAI-compatible error response.
pub fn validate_completion_fields_generic(
    request: &NvCreateCompletionRequest,
) -> Result<(), ErrorResponse> {
    request.validate().map_err(|e| {
        ErrorMessage::from_http_error(HttpError {
            code: 400,
            message: VALIDATION_PREFIX.to_string() + &e.to_string(),
        })
    })
}

/// OpenAI Responses input-token counting handler.
///
/// Handles `POST /v1/responses/input_tokens` and returns an estimated input
/// token count using a len/3 heuristic.
///
/// Like the Anthropic `/v1/messages/count_tokens` handler, this deliberately
/// performs neither a readiness nor a model-serving check: clients routinely
/// send routing names this frontend does not serve, and a pre-flight estimate
/// does not need a live model.
async fn handler_responses_input_tokens(
    State((_state, _template)): State<(Arc<service_v2::State>, Option<RequestTemplate>)>,
    headers: HeaderMap,
    body: Body,
) -> Result<Response, ErrorResponse> {
    let body = read_json_request_body(&headers, body).await?;
    let request: CountInputTokensRequest = parse_json_request("responses input_tokens", &body)?;
    Ok(Json(CountInputTokensResponse::new(request.estimate_tokens())).into_response())
}

/// OpenAI Responses Request Handler
///
/// This method will handle the incoming request for the /v1/responses endpoint.
async fn handler_responses(
    State((state, template)): State<(Arc<service_v2::State>, Option<RequestTemplate>)>,
    headers: HeaderMap,
    body: Body,
) -> Result<Response, ErrorResponse> {
    let body = read_json_request_body(&headers, body).await?;
    let mut request: NvCreateResponse = parse_json_request("responses", &body)?;

    // return a 503 if the service or model is not ready.
    // Resolve the templated model first so empty/missing `model` fields
    // don't bypass the gate.
    check_ready(&state)?;
    let resolved_model = resolve_request_model(
        request.inner.model.as_deref().unwrap_or(""),
        template.as_ref(),
    );
    if !resolved_model.is_empty() {
        check_model_serving_ready(&state, resolved_model)?;
    }

    if !state.nvext_enabled() {
        warn_nvext_disabled(
            "responses",
            request
                .nvext
                .as_ref()
                .is_some_and(CommonNvExt::has_non_cache_salt_fields)
                || has_non_cache_salt_routing_headers(&headers),
        );
    }
    request.nvext =
        apply_frontend_nvext_policy(request.nvext.take(), &headers, state.nvext_enabled());

    // create the context for the request
    let request_id = get_or_create_request_id(&headers);
    let streaming = request.inner.stream.unwrap_or(false);
    let raw_model = request.inner.model.as_deref().unwrap_or("");
    let resolved_model = resolve_request_model(raw_model, template.as_ref());
    // Canonicalize alias → primary for the metric label.
    let canonical_model = state.manager().resolve_canonical_name(resolved_model);
    let cancellation_labels = CancellationLabels {
        model: state
            .manager()
            .metric_model_for(&canonical_model)
            .to_string(),
        endpoint: Endpoint::Responses.to_string(),
        request_type: if streaming { "stream" } else { "unary" }.to_string(),
    };
    let mut request =
        context_from_headers_with_input_trigger(request, request_id, &headers, |request| {
            Some(classify_response_request(request))
        })?;
    if let Some(captured) = crate::request_trace::payload::capture_http_headers(&headers) {
        request.insert(
            crate::request_trace::payload::HTTP_HEADERS_CONTEXT_KEY,
            captured,
        );
    }
    let context = request.context();

    // create the connection handles
    let (mut connection_handle, stream_handle) = create_connection_monitor(
        context.clone(),
        Some(state.metrics_clone()),
        cancellation_labels,
    )
    .await;

    let response =
        tokio::spawn(responses(state, template, request, stream_handle).in_current_span())
            .await
            .map_err(|e| {
                ErrorMessage::internal_server_error_with_details(
                    "Failed to await responses task",
                    format!("{e:?}"),
                )
            })?;

    // if we got here, then we will return a response and the potentially long running task has completed successfully
    // without need to be cancelled.
    connection_handle.disarm();

    response
}

#[tracing::instrument(level = "debug", skip_all, fields(request_id = %request.id()))]
async fn responses(
    state: Arc<service_v2::State>,
    template: Option<RequestTemplate>,
    mut request: Context<NvCreateResponse>,
    stream_handle: ConnectionHandle,
) -> Result<Response, ErrorResponse> {
    // return a 503 if the service is not ready
    check_ready(&state)?;

    // Apply template values if present. When no template and no client-supplied
    // max_output_tokens, leave it as None for response echoing and let the
    // backend adapter compute the dynamic generation cap from its effective
    // prompt length.
    if let Some(template) = template {
        if request.inner.model.as_deref().unwrap_or("").is_empty() {
            request.inner.model = Some(template.model.clone());
        }
        if request.inner.temperature.is_none() {
            request.inner.temperature = Some(template.temperature);
        }
        if request.inner.max_output_tokens.is_none() {
            request.inner.max_output_tokens = Some(template.max_completion_tokens);
        }
    }
    tracing::trace!("Received responses request: {:?}", request.inner);

    // Resolve an alias to its primary served name and rewrite the request so
    // engine routing, metrics, and the response model all use the canonical
    // primary. The Responses API wraps model in Option<String>, so re-wrap
    // after resolution. Non-aliases pass through metric_model_for's guard.
    let original_model = request.inner.model.clone().unwrap_or_default();
    let canonical = state.manager().resolve_canonical_name(&original_model);
    if canonical != original_model {
        request.inner.model = Some(canonical.clone());
    }
    let model = canonical;
    let streaming = request.inner.stream.unwrap_or(false);
    let metric_model = state.manager().metric_model_for(&model).to_string();

    // Create http_queue_guard early - tracks time waiting to be processed
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);
    let mut inflight_guard = state.metrics_clone().create_inflight_guard(
        &metric_model,
        Endpoint::Responses,
        streaming,
        request.id(),
    );

    // Handle unsupported fields - if Some(resp) is returned by validate_unsupported_fields,
    // then a field was used that is unsupported. We will log an error message
    // and early return a 501 NOT_IMPLEMENTED status code.
    if let Some(resp) = validate_response_unsupported_fields(&request) {
        inflight_guard.mark_error(ErrorType::NotImplemented);
        return Ok(resp.into_response());
    }

    // Validate sampling and output parameters
    if let Err(err_response) = validate_responses_fields(&request) {
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        return Err(err_response);
    }

    // Extract request parameters before into_parts() consumes the request.
    // These are echoed back in the Response object per the OpenAI spec.
    let response_params = ResponseParams {
        model: request.inner.model.clone(),
        temperature: request.inner.temperature,
        top_p: request.inner.top_p,
        max_output_tokens: request.inner.max_output_tokens,
        parallel_tool_calls: request.inner.parallel_tool_calls,
        store: request.inner.store,
        tools: request.inner.tools.clone(),
        tool_choice: request.inner.tool_choice.clone(),
        instructions: request.inner.instructions.clone(),
        reasoning: request.inner.reasoning.clone(),
        text: request.inner.text.clone(),
        service_tier: request.inner.service_tier,
        include: request.inner.include.clone(),
        truncation: request.inner.truncation,
        // Upstream `CreateResponse` doesn't carry these yet; plumbed through so
        // the response serializer can default to 0.0 without hardcoding at the
        // build site. When upstream (or our shadow) adds the fields, sourcing
        // from the request becomes a one-line change here.
        presence_penalty: None,
        frequency_penalty: None,
        // Pass-through metadata — accepted on the request, echoed back on the
        // response so the caller can confirm receipt. Dynamo doesn't act on
        // these; see `validate_response_unsupported_fields` for rationale.
        prompt_cache_key: request.inner.prompt_cache_key.clone(),
        prompt_cache_retention: request.inner.prompt_cache_retention,
        safety_identifier: request.inner.safety_identifier.clone(),
    };
    let request_id = request.id().to_string();
    let (orig_request, context) = request.into_parts();

    let unified_request: UnifiedRequest = orig_request.try_into().map_err(|e: anyhow::Error| {
        tracing::error!(
            request_id,
            error = %e,
            "Failed to convert NvCreateResponse to UnifiedRequest",
        );
        let err_response = responses_conversion_error_response(e);
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;
    // Extract the API context before consuming the UnifiedRequest — this
    // carries Responses-specific fields (previous_response_id, store, etc.)
    // that the stream converter needs for faithful response reconstruction.
    let responses_ctx = unified_request.responses_context().cloned();
    let mut chat_request = unified_request.into_inner();
    if let Err(err_response) = normalize_chat_reasoning_template_args(&mut chat_request) {
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        return Err(err_response);
    }
    if let Err(error) = chat_request.validate() {
        let err_response = ErrorMessage::from_anyhow(
            invalid_argument(error.to_string()).into(),
            "Invalid responses request",
        );
        inflight_guard.mark_error(ErrorType::Validation);
        return Err(err_response);
    }

    // Always use internal streaming for aggregation.
    // Set stream_options.include_usage so the backend sends token counts in the final chunk.
    chat_request.inner.stream = Some(true);
    chat_request.inner.stream_options =
        Some(dynamo_protocols::types::ChatCompletionStreamOptions {
            include_usage: true,
            continuous_usage_stats: false,
        });

    let mut request = context.map(|mut _req| chat_request);
    if response_params.max_output_tokens.is_none() {
        request.insert(PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY, true);
    }

    tracing::trace!("Getting chat completions engine for model: {}", model);

    let (engine, parsing_options) = state
        .manager()
        .get_chat_completions_engine_with_parsing(&model)
        .map_err(|e| {
            let err_response = ErrorMessage::from_model_error(&e);
            inflight_guard.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    // The Responses API is converted to the same chat request contract. Narrow
    // the model parser before unary aggregation just as the streaming path does.
    let parsing_options = apply_request_tool_call_parsing_options(parsing_options, &request)
        .map_err(|e| {
            let err_response = ErrorMessage::from_anyhow(e.into(), "Invalid tool_choice");
            inflight_guard.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    // Responses requests share the chat-completions aggregator for the unary
    // path. Thread this option through so its post-parse fallback also caps a
    // model-produced batch to the first tool call when parallel calls are
    // disabled. The streaming Responses converter enforces the same contract.
    let parsing_options =
        parsing_options.with_parallel_tool_calls(request.inner.parallel_tool_calls);

    // A non-streaming Responses request DOES reach the aggregator (forcing
    // stream=true on the converted request only drives internal streaming; the
    // client-facing `streaming` flag still selects the aggregating branch
    // below), so it needs the same backstop the chat path installs.
    //
    // This used to be left unset, which was safe only because the
    // Responses-to-chat conversion hard-coded `chat_template_args: None` and
    // `force_nonempty_content` could never be set on this path. Now that the
    // conversion forwards those args, the flag has to be wired here: the
    // streaming stage gates on the request's own args via
    // `wants_reasoning_as_content_when_empty`, but the aggregating branch reads
    // this flag instead.
    let move_reasoning_to_content_when_empty =
        crate::preprocessor::OpenAIPreprocessor::wants_reasoning_as_content_when_empty(
            request.chat_template_args.as_ref(),
        );
    let parsing_options = parsing_options
        .with_move_reasoning_to_content_when_empty(move_reasoning_to_content_when_empty);

    // Computed before `request` moves into `generate`. Responses streams use
    // the same force-nonempty deferral as chat completions and therefore need
    // the same fallback keep-alive when every data frame may be withheld.
    let stream_can_defer_all_output =
        request_stream_can_defer_all_output(&parsing_options, request.chat_template_args.as_ref());

    let mut response_collector = state
        .metrics_clone()
        .create_response_collector(&metric_model);

    tracing::trace!("Issuing generate call for responses");

    // issue the generate call on the engine
    let engine_stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model, super::metrics::Endpoint::Responses);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate completions");
        inflight_guard.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    // Capture the context to cancel the stream if the client disconnects
    let ctx = engine_stream.context();

    if streaming {
        // Peek the first non-annotation event for a synchronous backend error
        // before committing HTTP 200 — same rationale as chat_completions
        // above. Short peek window; the long backend-inactivity safety net
        // lives in `monitor_for_disconnects`.
        let engine_stream = match pre_commit_error_peek_timeout() {
            Some(dur) => check_for_backend_error(engine_stream, Some(dur))
                .await
                .map_err(|err_response| {
                    tracing::error!(request_id = %request_id, "Backend error detected: {:?}", err_response);
                    inflight_guard.mark_error(extract_error_type_from_response(&err_response));
                    err_response
                })?,
            // Env var unset → skip peek, commit HTTP 200 immediately.
            None => Box::pin(engine_stream)
                as std::pin::Pin<
                    Box<dyn futures::Stream<Item = _> + Send>,
                >,
        };

        // Streaming path: convert chat completion stream chunks to Responses API SSE events.
        // The engine yields Annotated<NvCreateChatCompletionStreamResponse>. We extract the
        // inner stream response data and convert it to Responses API events.
        use crate::protocols::openai::responses::stream_converter::ResponseStreamConverter;

        let mut converter = match responses_ctx {
            Some(ctx) => ResponseStreamConverter::with_context(model.clone(), response_params, ctx),
            None => ResponseStreamConverter::new(model.clone(), response_params),
        };

        let mut http_queue_guard = Some(http_queue_guard);
        let error_signal = StreamErrorSignal::default();
        let producer_error_signal = error_signal.clone();

        let mut engine_stream = Box::pin(engine_stream);
        let full_stream = async_stream::stream! {
            let mut events = Vec::with_capacity(4);
            converter.append_start_events(&mut events);
            for event in events.drain(..) {
                yield event.map_err(axum::Error::new);
            }

            // Preserve the first backend error for the terminal Responses event.
            let mut backend_error = None;

            while let Some(annotated_chunk) = engine_stream.next().await {
                process_chat_response_and_observe_metrics(
                    &annotated_chunk,
                    &mut response_collector,
                    &mut http_queue_guard,
                );

                if let Some(backend_error_info) =
                    extract_backend_error_if_present(&annotated_chunk)
                {
                    let error_response = backend_error_response(backend_error_info);
                    producer_error_signal
                        .set(extract_error_type_from_response(&error_response));
                    backend_error.get_or_insert_with(|| ErrorObject {
                        code: responses_error_code(error_response.0).to_string(),
                        message: error_response.1.message.clone(),
                    });
                    continue;
                }

                let Some(stream_resp) = annotated_chunk.data else {
                    continue;
                };

                converter.append_chunk_events(&stream_resp, &mut events);
                for event in events.drain(..) {
                    yield event.map_err(axum::Error::new);
                }
            }

            if let Some(error) = backend_error {
                let terminal_event = converter.append_error_events(error, &mut events);
                for event in events.drain(..) {
                    yield event.map_err(axum::Error::new);
                }
                if terminal_event.is_ok() {
                    // From this yield onward, response.failed is sufficient for
                    // a client to stop consuming without being a disconnect.
                    producer_error_signal.mark_terminal_event_emitted();
                }
                yield terminal_event.map_err(axum::Error::new);
            } else {
                converter.append_end_events(&mut events);
                for event in events.drain(..) {
                    yield event.map_err(axum::Error::new);
                }
            }
        };

        // Wrap with disconnect monitoring: detects client disconnects, cancels generation,
        // and defers inflight_guard.mark_ok() until the stream completes.
        let stream = monitor_for_disconnects_with_error_signal(
            full_stream,
            ctx,
            inflight_guard,
            stream_handle,
            error_signal,
        );

        let mut sse_stream = Sse::new(stream);
        if let Some(keep_alive) = state.sse_keep_alive_for_response(stream_can_defer_all_output) {
            sse_stream = sse_stream.keep_alive(KeepAlive::default().interval(keep_alive));
        }

        Ok(sse_stream.into_response())
    } else {
        // Non-streaming path: aggregate stream into single response

        // Check first event for backend errors before aggregating (non-streaming only)
        let stream_with_check =
            check_for_backend_error(engine_stream, None)
                .await
                .map_err(|error_response| {
                    tracing::error!(request_id, "Backend error detected: {:?}", error_response);
                    inflight_guard.mark_error(extract_error_type_from_response(&error_response));
                    error_response
                })?;

        let mut http_queue_guard = Some(http_queue_guard);
        let stream = stream_with_check.inspect(move |response| {
            process_chat_response_and_observe_metrics(
                response,
                &mut response_collector,
                &mut http_queue_guard,
            );
        });

        let response =
            NvCreateChatCompletionResponse::from_annotated_stream(stream, parsing_options.clone())
                .await
                .map_err(|e| {
                    tracing::error!(request_id, "Failed to fold responses stream: {:?}", e);
                    let err_response =
                        ErrorMessage::internal_server_error("Failed to fold responses stream");
                    inflight_guard.mark_error(extract_error_type_from_response(&err_response));
                    err_response
                })?;

        // Convert NvCreateChatCompletionResponse --> NvResponse
        let response: NvResponse =
            chat_completion_to_response(response, &response_params, responses_ctx.as_ref())
                .map_err(|e| {
                    tracing::error!(
                        request_id,
                        "Failed to convert NvCreateChatCompletionResponse to NvResponse: {:?}",
                        e
                    );
                    let err_response =
                        ErrorMessage::internal_server_error("Failed to convert internal response");
                    inflight_guard.mark_error(extract_error_type_from_response(&err_response));
                    err_response
                })?;

        inflight_guard.mark_ok();
        // If the engine context was killed (client disconnect), the response was
        // assembled but never delivered. Override to cancelled.
        if ctx.is_killed() {
            inflight_guard.mark_error(ErrorType::Cancelled);
        }

        Ok(Json(response).into_response())
    }
}

/// Checks for unsupported fields in the request.
/// Returns Some(response) if unsupported fields are present.
pub fn validate_response_unsupported_fields(
    request: &NvCreateResponse,
) -> Option<impl IntoResponse> {
    let inner = &request.inner;

    if let Some(field) = request
        .nvext
        .as_ref()
        .and_then(|nvext| nvext.extra_fields.as_ref())
        .and_then(|fields| {
            fields
                .iter()
                .find(|field| matches!(field.as_str(), "completion_token_ids" | "prompt_logprobs"))
        })
    {
        return Some(ErrorMessage::not_implemented_error(format!(
            "{VALIDATION_PREFIX}`nvext.extra_fields=[\"{field}\"]` is not supported by the Responses API."
        )));
    }

    if inner.background == Some(true) {
        return Some(ErrorMessage::not_implemented_error(
            VALIDATION_PREFIX.to_string() + "`background: true` is not supported.",
        ));
    }
    if inner.previous_response_id.is_some() {
        return Some(ErrorMessage::not_implemented_error(
            VALIDATION_PREFIX.to_string() + "`previous_response_id` is not supported.",
        ));
    }
    if inner.prompt.is_some() {
        return Some(ErrorMessage::not_implemented_error(
            VALIDATION_PREFIX.to_string() + "`prompt` is not supported.",
        ));
    }
    // Reject directive fields that change semantics if silently dropped.
    // `max_tool_calls` is a hard cap on tool invocations — accepting it
    // without enforcement would let a caller send `max_tool_calls: 5` and
    // see `max_tool_calls: null` in the response, assuming their limit was
    // honored. Fail loud until real enforcement lands.
    //
    // Pass-through metadata fields (`prompt_cache_key`,
    // `prompt_cache_retention`, `safety_identifier`) are deliberately
    // accepted and echoed back on the response instead. They're hints for
    // OpenAI's caching/moderation backends, not directives — Codex sends
    // `prompt_cache_key` on every request — and the OpenResponses spec
    // includes them on the response body, so echoing the caller's value
    // makes receipt observable without needing a real backend.
    if inner.max_tool_calls.is_some() {
        return Some(ErrorMessage::not_implemented_error(
            VALIDATION_PREFIX.to_string() + "`max_tool_calls` is not supported.",
        ));
    }
    None
}

/// Validates sampling and output parameters on the Responses API request.
pub fn validate_responses_fields(request: &NvCreateResponse) -> Result<(), ErrorResponse> {
    use crate::protocols::openai::validate;

    let map_err = |e: anyhow::Error| {
        ErrorMessage::from_http_error(HttpError {
            code: 400,
            message: VALIDATION_PREFIX.to_string() + &e.to_string(),
        })
    };

    validate::validate_temperature(request.inner.temperature).map_err(&map_err)?;
    validate::validate_top_p(request.inner.top_p).map_err(&map_err)?;
    validate::validate_max_tokens(request.inner.max_output_tokens).map_err(&map_err)?;

    if let Some(text) = &request.inner.text {
        use crate::protocols::openai::responses::convert_text_format;
        if let Some(response_format) = convert_text_format(text) {
            validate::validate_response_format(&Some(response_format)).map_err(&map_err)?;
        }
    }

    Ok(())
}

// todo - abstract this to the top level lib.rs to be reused
pub(crate) fn check_ready(state: &Arc<service_v2::State>) -> Result<(), ErrorResponse> {
    if !state.is_ready() {
        return Err(ErrorMessage::_service_unavailable());
    }
    Ok(())
}

/// Returns an OpenAI-compatible JSON `404` error response for an
/// unmatched route.
pub(crate) fn unmatched_route_response(method: &Method, uri: &Uri) -> ErrorResponse {
    let code = StatusCode::NOT_FOUND;
    (
        code,
        Json(ErrorMessage {
            message: format!("Route not found: {} {}", method, uri.path()),
            error_type: map_error_code_to_error_type(code),
            code: code.as_u16(),
            details: None,
            metric_error_type: None,
        }),
    )
}

/// Canonical, customer-facing message for "model is registered but not yet
/// ready to serve requests" (deployment still initializing or incomplete).
///
/// One message for every not-ready cause — whichever worker role is missing,
/// the client sees the same text. Deliberately free of internal taxonomy
/// (worker types, namespaces, "worker set"): it stays clear and actionable for
/// end users without leaking deployment internals. Operators get the detailed,
/// per-role breakdown from `GET /v1/models/{model}/ready` instead.
pub(crate) fn model_not_ready_message(model_name: &str) -> String {
    format!(
        "Model `{model_name}` is not ready to serve requests yet. \
         The deployment may still be starting up or is not fully provisioned. \
         Please retry shortly."
    )
}

/// Per-model serving readiness gate.
///
/// Composes AND-wise with [`check_ready`]: a request is admitted only when
/// (a) the process is ready (`check_ready`) AND (b) at least one namespace
/// for this specific model has a complete set of workers — every worker's
/// `needs` DNF is satisfied by the worker types currently present in that
/// namespace.
///
/// Returns `503 Service Unavailable` with a structured body when the model
/// isn't ready to serve. Models absent from the committed catalog, including
/// discovered models still being built, fall through here; the per-handler
/// engine lookup later in the request path returns a 404 instead.
pub(crate) fn check_model_serving_ready(
    state: &Arc<service_v2::State>,
    model_name: &str,
) -> Result<(), ErrorResponse> {
    let Some(model) = state.manager().get_committed_model(model_name) else {
        // Not committed — let the per-endpoint engine accessor produce the
        // canonical 404. The readiness gate has nothing to say.
        return Ok(());
    };
    if model.has_ready_workers() {
        return Ok(());
    }
    Err(ErrorMessage::service_unavailable_with_body(
        model_not_ready_message(model_name),
    ))
}

/// openai compatible format
/// Example:
/// {
///  "object": "list",
///  "data": [
///    {
///      "id": "model-id-0",
///      "object": "model",
///      "created": 1686935002,
///      "owned_by": "organization-owner"
///    },
///    ]
/// }
async fn list_models_openai(
    State(state): State<Arc<service_v2::State>>,
) -> Result<Response, ErrorResponse> {
    check_ready(&state)?;

    let created = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap()
        .as_secs();

    // Build context_length lookup from model deployment cards
    let cards = state.manager().get_model_cards();
    let card_map: HashMap<String, u32> = cards
        .iter()
        .map(|c| (c.display_name.clone(), c.effective_context_length()))
        .collect();

    // Env var overrides (take precedence over MDC values)
    let cw_override: Option<u64> = std::env::var("DYN_CONTEXT_WINDOW")
        .ok()
        .and_then(|v| v.parse().ok());
    let mot_override: Option<u64> = std::env::var("DYN_MAX_OUTPUT_TOKENS")
        .ok()
        .and_then(|v| v.parse().ok());

    let mut data = Vec::new();

    // Only list models whose worker set is complete in at least one namespace.
    // A registered-but-broken deployment (e.g. decode-only with no prefill peer)
    // is hidden until a peer joins.
    let models: HashSet<String> = state.manager().serving_ready_display_names();
    for model_name in models {
        // Alias entries have no card of their own (keyed by the primary's
        // display_name); fall back to the primary's context length.
        let context_window = cw_override.or_else(|| {
            card_map
                .get(&model_name)
                .or_else(|| card_map.get(&state.manager().resolve_canonical_name(&model_name)))
                .map(|&cl| cl as u64)
        });
        data.push(ModelListing {
            id: model_name.clone(),
            object: "model",
            created,
            owned_by: "nvidia".to_string(),
            context_window,
            max_output_tokens: mot_override,
        });
    }

    let out = ListModelOpenAI {
        object: "list",
        data,
    };
    Ok(Json(out).into_response())
}

#[derive(Serialize)]
struct ListModelOpenAI {
    object: &'static str, // always "list"
    data: Vec<ModelListing>,
}

#[derive(Serialize)]
struct ModelListing {
    id: String,
    object: &'static str, // always "model" per OpenAI spec
    created: u64,         // Seconds since epoch
    owned_by: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    context_window: Option<u64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    max_output_tokens: Option<u64>,
}

/// Create an Axum [`Router`] for the OpenAI API Completions endpoint
/// If not path is provided, the default path is `/v1/completions`
pub fn completions_router(
    state: Arc<service_v2::State>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let path = path.unwrap_or("/v1/completions".to_string());
    let doc = RouteDoc::new(axum::http::Method::POST, &path);
    let router = Router::new()
        .route(&path, post(handler_completions))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state(state);
    (vec![doc], router)
}

/// Create an Axum [`Router`] for the OpenAI API Chat Completions endpoint
/// If not path is provided, the default path is `/v1/chat/completions`
pub fn chat_completions_router(
    state: Arc<service_v2::State>,
    template: Option<RequestTemplate>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let path = path.unwrap_or("/v1/chat/completions".to_string());
    let doc = RouteDoc::new(axum::http::Method::POST, &path);
    let router = Router::new()
        .route(&path, post(handler_chat_completions))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state((state, template));
    (vec![doc], router)
}

/// Create an Axum [`Router`] for the OpenAI API Embeddings endpoint
/// If not path is provided, the default path is `/v1/embeddings`
pub fn embeddings_router(
    state: Arc<service_v2::State>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let path = path.unwrap_or("/v1/embeddings".to_string());
    let doc = RouteDoc::new(axum::http::Method::POST, &path);
    let router = Router::new()
        .route(&path, post(embeddings))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state(state);
    (vec![doc], router)
}

/// Create an Axum [`Router`] for the `/v1/classify` endpoint (sequence
/// classification / cross-encoder pooling). If no path is provided, the
/// default path is `/v1/classify`. Deployments migrating clients from native
/// `vllm-serve` (which mounts a bare `/classify`) can set the path via
/// `DYN_HTTP_SVC_CLASSIFY_PATH`.
pub fn classify_router(
    state: Arc<service_v2::State>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let path = path.unwrap_or("/v1/classify".to_string());
    let doc = RouteDoc::new(axum::http::Method::POST, &path);
    let router = Router::new()
        .route(&path, post(classify))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state(state);
    (vec![doc], router)
}

/// Create an Axum [`Router`] for the `/v1/pooling` endpoint (raw pooler output
/// from pooling-runner models). If no path is provided, the default path is
/// `/v1/pooling`. Deployments migrating clients from native `vllm-serve`
/// (which mounts a bare `/pooling`) can set the path via
/// `DYN_HTTP_SVC_POOLING_PATH`.
pub fn pooling_router(
    state: Arc<service_v2::State>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let path = path.unwrap_or("/v1/pooling".to_string());
    let doc = RouteDoc::new(axum::http::Method::POST, &path);
    let router = Router::new()
        .route(&path, post(pooling))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state(state);
    (vec![doc], router)
}

/// Create an Axum [`Router`] for the OpenAI Batch API skeleton.
///
/// The first slice exposes the route and protocol shape. Durable file storage,
/// batch job persistence, dispatch, and output assembly are implemented by
/// follow-up work, so handlers return explicit 501 responses instead of
/// accepting work that cannot complete yet.
pub fn batch_router(
    state: Arc<service_v2::State>,
    files_path: Option<String>,
    batches_path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let files_path = files_path.unwrap_or("/v1/files".to_string());
    let file_content_path = format!("{}/{{file_id}}/content", files_path);
    let batches_path = batches_path.unwrap_or("/v1/batches".to_string());
    let batch_path = format!("{}/{{batch_id}}", batches_path);

    let docs = vec![
        RouteDoc::new(axum::http::Method::POST, &files_path),
        RouteDoc::new(axum::http::Method::GET, &file_content_path),
        RouteDoc::new(axum::http::Method::POST, &batches_path),
        RouteDoc::new(axum::http::Method::GET, &batch_path),
    ];

    let router = Router::new()
        .route(&files_path, post(create_batch_file))
        .route(&file_content_path, get(retrieve_batch_file_content))
        .route(&batches_path, post(create_batch))
        .route(&batch_path, get(retrieve_batch))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state(state);

    (docs, router)
}

async fn create_batch_file() -> Result<Response, ErrorResponse> {
    Err(ErrorMessage::not_implemented_error(
        BATCH_FILE_STORAGE_NOT_IMPLEMENTED,
    ))
}

async fn create_batch() -> Result<Response, ErrorResponse> {
    Err(ErrorMessage::not_implemented_error(
        BATCH_JOB_STATE_NOT_IMPLEMENTED,
    ))
}

async fn retrieve_batch(
    axum::extract::Path(_batch_id): axum::extract::Path<String>,
) -> Result<Response, ErrorResponse> {
    Err(ErrorMessage::not_implemented_error(
        BATCH_JOB_STATE_NOT_IMPLEMENTED,
    ))
}

async fn retrieve_batch_file_content(
    axum::extract::Path(_file_id): axum::extract::Path<String>,
) -> Result<Response, ErrorResponse> {
    Err(ErrorMessage::not_implemented_error(
        BATCH_OUTPUT_RETRIEVAL_NOT_IMPLEMENTED,
    ))
}

/// List Models
pub fn list_models_router(
    state: Arc<service_v2::State>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    // Standard OpenAI compatible list models endpoint
    let openai_path = path.unwrap_or("/v1/models".to_string());
    let retrieve_path = format!("{}/{{*model_id}}", openai_path);
    let doc_for_openai = RouteDoc::new(axum::http::Method::GET, &openai_path);
    let doc_for_retrieve = RouteDoc::new(axum::http::Method::GET, &retrieve_path);
    // Doc-only: the readiness sub-resource is served by `get_model_openai` via
    // the catch-all retrieve route above (a wildcard must be the terminal
    // segment, so it can't be its own axum route). Advertised for discovery.
    let doc_for_readiness = RouteDoc::new(
        axum::http::Method::GET,
        format!("{}/{{model_id}}/ready", openai_path),
    );

    let router = Router::new()
        .route(&openai_path, get(list_models_openai))
        .route(&retrieve_path, get(get_model_openai))
        .with_state(state);

    (
        vec![doc_for_openai, doc_for_retrieve, doc_for_readiness],
        router,
    )
}

/// Retrieve a single model by ID (OpenAI format).
///
/// Per the OpenAI API spec: `GET /v1/models/{model}` returns a model object.
/// Uses wildcard path to support model IDs with slashes (e.g. `Qwen/Qwen3.5-35B-A3B-FP8`).
async fn get_model_openai(
    State(state): State<Arc<service_v2::State>>,
    axum::extract::Path(model_id): axum::extract::Path<String>,
) -> Result<Response, ErrorResponse> {
    check_ready(&state)?;

    let model_id = model_id.strip_prefix('/').unwrap_or(&model_id);

    // The retrieve route (`/v1/models/{*model_id}`) is a catch-all, so model
    // IDs can contain '/' — and may even end in '/ready'. We therefore
    // dispatch by precedence: an *exact* model match always wins, and only when
    // there is no such model do we treat a trailing `/ready` as the
    // per-model readiness sub-resource (Mechanism 4). This means a model
    // literally named `foo/ready` is still retrievable and never shadowed.
    //
    // Exact match is resolved against every committed model, not just the
    // displayable ones, so a committed-but-not-yet-ready `foo/ready` still
    // wins over the readiness sub-resource of a sibling `foo`.
    // `get_model_retrieve` applies the readiness gate itself (503 if not ready).
    if state.manager().get_committed_model(model_id).is_some() {
        return get_model_retrieve(&state, model_id);
    }

    // Readiness sub-resource. Resolves against all committed models (above
    // exact check failed, so `model_id` is not itself a committed model);
    // the whole point of this endpoint is to diagnose committed models that
    // are not yet ready, so it must find them too.
    if let Some(base) = model_id.strip_suffix("/ready")
        && state.manager().get_committed_model(base).is_some()
    {
        return get_model_readiness(&state, base);
    }

    Err(ErrorMessage::model_not_found())
}

/// `GET /v1/models/{model}` — the OpenAI retrieve-model object. Reports the
/// model only if it is ready to serve (mirrors the `list_models_openai` filter).
fn get_model_retrieve(
    state: &Arc<service_v2::State>,
    model_id: &str,
) -> Result<Response, ErrorResponse> {
    check_model_serving_ready(state, model_id)?;

    let created = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap()
        .as_secs();

    // Alias entries have no card of their own (cards are keyed by the primary's
    // display_name); fall back to the primary so an alias reports the same
    // context_window that `GET /v1/models` lists for it.
    let canonical_model = state.manager().resolve_canonical_name(model_id);
    let cards = state.manager().get_model_cards();
    let context_length = cards
        .iter()
        .find(|c| c.display_name == model_id || c.display_name == canonical_model)
        .map(|c| c.effective_context_length() as u64);
    let context_window: Option<u64> = std::env::var("DYN_CONTEXT_WINDOW")
        .ok()
        .and_then(|v| v.parse().ok())
        .or(context_length);
    let max_output_tokens: Option<u64> = std::env::var("DYN_MAX_OUTPUT_TOKENS")
        .ok()
        .and_then(|v| v.parse().ok());

    Ok(Json(ModelListing {
        id: model_id.to_string(),
        object: "model",
        created,
        owned_by: "nvidia".to_string(),
        context_window,
        max_output_tokens,
    })
    .into_response())
}

/// `GET /v1/models/{model}/ready` — structured per-namespace worker readiness
/// detail (Mechanism 4). Deliberately *not* readiness-gated: it exists to
/// diagnose models that are not yet ready, so it returns 200 with the full
/// breakdown regardless of whether the model would be served.
fn get_model_readiness(
    state: &Arc<service_v2::State>,
    model_id: &str,
) -> Result<Response, ErrorResponse> {
    let model = state
        .manager()
        .get_committed_model(model_id)
        .ok_or_else(ErrorMessage::model_not_found)?;
    Ok(Json(model.namespace_readiness()).into_response())
}

/// Create an Axum [`Router`] for the OpenAI API Responses endpoints
/// (`/v1/responses` and `/v1/responses/input_tokens`).
/// If not path is provided, the default path is `/v1/responses`
pub fn responses_router(
    state: Arc<service_v2::State>,
    template: Option<RequestTemplate>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let path = path.unwrap_or("/v1/responses".to_string());
    // Derive the subroute from the parent with any trailing slash trimmed.
    // `DYN_HTTP_SVC_RESPONSES_PATH=/custom/` is a working configuration for the
    // parent — axum matches `POST /custom/` — but naively appending would
    // register `/custom//input_tokens`, and axum does not treat that as
    // equivalent to the `/custom/input_tokens` a client would actually call.
    // The parent is registered verbatim, so trimming here changes only the
    // derived path and leaves existing configurations behaving as they do now.
    let input_tokens_path = format!("{}/input_tokens", path.trim_end_matches('/'));
    let doc = RouteDoc::new(axum::http::Method::POST, &path);
    let input_tokens_doc = RouteDoc::new(axum::http::Method::POST, &input_tokens_path);
    let router = Router::new()
        .route(&path, post(handler_responses))
        .route(&input_tokens_path, post(handler_responses_input_tokens))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state((state, template));
    (vec![doc, input_tokens_doc], router)
}

async fn images(
    State(state): State<Arc<service_v2::State>>,
    headers: HeaderMap,
    Json(mut request): Json<NvCreateImageRequest>,
) -> Result<Response, ErrorResponse> {
    // return a 503 if the service is not ready
    // (per-model readiness check is deferred until after we resolve the
    // ImageModel enum into a string; see below)
    check_ready(&state)?;

    request.nest_passthrough();
    let request_id = get_or_create_request_id(&headers);
    let request = context_from_headers(request, request_id, &headers)?;
    let request_id = request.id().to_string();

    // Images are typically not streamed, so we default to non-streaming
    let streaming = false;

    // Get the model name from the request (diffusion model)
    let model = request
        .inner
        .model
        .as_ref()
        .map(|m| match m {
            dynamo_protocols::types::ImageModel::DallE2 => "dall-e-2".to_string(),
            dynamo_protocols::types::ImageModel::DallE3 => "dall-e-3".to_string(),
            dynamo_protocols::types::ImageModel::GptImage1 => "gpt-image-1".to_string(),
            dynamo_protocols::types::ImageModel::GptImage1dot5 => "gpt-image-1.5".to_string(),
            dynamo_protocols::types::ImageModel::GptImage1Mini => "gpt-image-1-mini".to_string(),
            dynamo_protocols::types::ImageModel::GptImage2 => "gpt-image-2".to_string(),
            dynamo_protocols::types::ImageModel::Other(s) => s.clone(),
        })
        .unwrap_or_else(|| "diffusion".to_string());

    // Per-model serving readiness gate (now that we have a resolved model
    // name string).
    check_model_serving_ready(&state, &model)?;

    let metric_model = state.manager().metric_model_for(&model).to_string();

    // Create http_queue_guard early - tracks time waiting to be processed
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    // Get the image generation engine
    let engine = state
        .manager()
        .get_images_engine(&model)
        .map_err(|e| ErrorMessage::from_model_error(&e))?;

    // this will increment the inflight gauge for the model
    let mut inflight = state.metrics_clone().create_inflight_guard(
        &model,
        Endpoint::Images,
        streaming,
        &request_id,
    );

    let mut response_collector = state.metrics_clone().create_response_collector(&model);

    // Issue the generate call on the engine
    // Note: This uses ServerStreamingEngine for internal routing/distribution,
    // NOT for client-facing SSE streaming. The stream is immediately folded into
    // a single response below.
    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model, super::metrics::Endpoint::Images);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate images");
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    // Process stream to collect metrics and drop http_queue_guard on first response
    let mut http_queue_guard = Some(http_queue_guard);
    let stream = stream.inspect(move |response| {
        // Calls observe_response() on each item - drops http_queue_guard on first item
        process_response_and_observe_metrics(
            response,
            &mut response_collector,
            &mut http_queue_guard,
        );
    });

    // Images are returned as a single response (non-streaming to client)
    // Fold the internal stream into a single response
    let response = NvImagesResponse::from_annotated_stream(stream)
        .await
        .map_err(|e| {
            tracing::error!("Failed to fold images stream for {}: {:?}", request_id, e);
            let err_response = ErrorMessage::internal_server_error("Failed to fold images stream");
            inflight.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    inflight.mark_ok();
    Ok(Json(response).into_response())
}

/// Handler for `/v1/images/edits` (I2I). Requires `input_reference`.
async fn images_edits(
    state: State<Arc<service_v2::State>>,
    headers: HeaderMap,
    Json(request): Json<NvCreateImageRequest>,
) -> Result<Response, ErrorResponse> {
    if request.input_reference.is_none() {
        let code = StatusCode::BAD_REQUEST;
        return Err((
            code,
            Json(ErrorMessage {
                message: "input_reference is required for /v1/images/edits".to_string(),
                error_type: map_error_code_to_error_type(code),
                code: code.as_u16(),
                details: None,
                metric_error_type: None,
            }),
        ));
    }
    images(state, headers, Json(request)).await
}

/// Create an Axum [`Router`] for the OpenAI API Images endpoints.
/// `/v1/images/generations` accepts optional `input_reference` (T2I or TI2I).
/// `/v1/images/edits` requires `input_reference` (I2I).
pub fn images_router(
    state: Arc<service_v2::State>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let generations_path = path.unwrap_or("/v1/images/generations".to_string());
    let edits_path = generations_path.replace("/generations", "/edits");
    let doc = RouteDoc::new(axum::http::Method::POST, &generations_path);
    let edits_doc = RouteDoc::new(axum::http::Method::POST, &edits_path);
    let router = Router::new()
        .route(&generations_path, post(images))
        .route(&edits_path, post(images_edits))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state(state);
    (vec![doc, edits_doc], router)
}

async fn videos(
    State(state): State<Arc<service_v2::State>>,
    headers: HeaderMap,
    Json(mut request): Json<NvCreateVideoRequest>,
) -> Result<Response, ErrorResponse> {
    // return a 503 if the service or model is not ready
    check_ready(&state)?;
    check_model_serving_ready(&state, &request.model)?;

    request.nest_passthrough();
    let request_id = get_or_create_request_id(&headers);
    let request = context_from_headers(request, request_id, &headers)?;
    let request_id = request.id().to_string();

    let streaming = request.stream.unwrap_or(false);

    // Get the model name from the request (video generation model)
    let model = request.model.clone();
    let metric_model = state.manager().metric_model_for(&model).to_string();

    // Create http_queue_guard early - tracks time waiting to be processed
    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    // Get the video generation engine
    let engine = state
        .manager()
        .get_videos_engine(&model)
        .map_err(|e| ErrorMessage::from_model_error(&e))?;

    // this will increment the inflight gauge for the model
    let mut inflight = state.metrics_clone().create_inflight_guard(
        &model,
        Endpoint::Videos,
        streaming,
        &request_id,
    );

    let mut response_collector = state.metrics_clone().create_response_collector(&model);

    // issue the generate call on the engine
    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model, super::metrics::Endpoint::Videos);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate videos");
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    let mut http_queue_guard = Some(http_queue_guard);

    if streaming {
        // [gluo TODO] revisit the cancellation handling here,
        // should be unified with chat_completions.
        let ctx = stream.context();
        let (mut connection_handle, stream_handle) = create_connection_monitor(
            ctx.clone(),
            Some(state.metrics_clone()),
            CancellationLabels {
                model: model.clone(),
                endpoint: Endpoint::Videos.to_string(),
                request_type: "stream".to_string(),
            },
        )
        .await;
        let stream = stream.flat_map(move |response| {
            let sse_result = process_response_using_event_converter_and_observe_metrics(
                EventConverter::from(response),
                &mut response_collector,
                &mut http_queue_guard,
            );
            match sse_result {
                Ok(Some(ev)) => stream::iter(vec![Ok(ev)]),
                Ok(None) => stream::iter(vec![]),
                Err(e) => stream::iter(vec![Err(e)]),
            }
        });
        // monitor_for_disconnects: arms stream_handle, pre-marks inflight Cancelled,
        // emits data:[DONE] on natural end, demotes to Internal on mid-stream Err,
        // and kills the engine context when the client disconnects.
        let stream = monitor_for_disconnects(stream, ctx, inflight, stream_handle);

        let mut sse_stream = Sse::new(stream);
        if let Some(keep_alive) = state.sse_keep_alive() {
            sse_stream = sse_stream.keep_alive(KeepAlive::default().interval(keep_alive));
        }
        // Disarm immediately: we return the body directly, so disconnect detection
        // transfers to stream_handle (armed inside monitor_for_disconnects).
        connection_handle.disarm();
        Ok(sse_stream.into_response())
    } else {
        let stream = stream.inspect(move |response| {
            process_response_and_observe_metrics(
                response,
                &mut response_collector,
                &mut http_queue_guard,
            );
        });

        let response = NvVideosResponse::from_annotated_stream(stream)
            .await
            .map_err(|e| {
                tracing::error!("Failed to fold videos stream for {}: {:?}", request_id, e);
                let err_response =
                    ErrorMessage::internal_server_error("Failed to fold videos stream");
                inflight.mark_error(extract_error_type_from_response(&err_response));
                err_response
            })?;

        inflight.mark_ok();
        Ok(Json(response).into_response())
    }
}

/// [EXPERIMENTAL] MJPEG streaming handler for `/v1/videos/stream`.
///
/// The backend is expected to yield one [`NvVideosResponse`] per frame, carrying a
/// JPEG-encoded frame as `data[0].b64_json`. This handler decodes each frame and
/// writes it as an MJPEG multipart boundary so the client receives a live
/// `multipart/x-mixed-replace` stream viewable directly in a browser `<img>` tag
/// or via `ffplay http://.../v1/videos/stream`.
async fn video_stream(
    State(state): State<Arc<service_v2::State>>,
    headers: HeaderMap,
    Json(mut request): Json<NvCreateVideoRequest>,
) -> Result<Response, ErrorResponse> {
    check_ready(&state)?;
    check_model_serving_ready(&state, &request.model)?;

    request.nest_passthrough();
    let request_id = get_or_create_request_id(&headers);
    let request = context_from_headers(request, request_id, &headers)?;
    let model = request.model.clone();
    let metric_model = state.manager().metric_model_for(&model).to_string();

    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    let engine = state
        .manager()
        .get_videos_engine(&model)
        .map_err(|e| ErrorMessage::from_model_error(&e))?;

    let mut inflight =
        state
            .metrics_clone()
            .create_inflight_guard(&model, Endpoint::Videos, true, request.id());

    let mut response_collector = state.metrics_clone().create_response_collector(&model);

    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&model, super::metrics::Endpoint::Videos);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to start video stream");
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    // Capture the context to cancel the stream if the client disconnects.
    let ctx = stream.context();

    // Create connection monitor. The connection_handle is disarmed immediately because
    // video_stream returns the streaming body directly (graceful handler exit).
    // The stream_handle is armed below and lives inside the monitored stream so that
    // a client disconnect (body drop) signals the engine context to cancel.
    let (mut connection_handle, mut stream_handle) = create_connection_monitor(
        ctx.clone(),
        Some(state.metrics_clone()),
        CancellationLabels {
            model: model.clone(),
            endpoint: Endpoint::Videos.to_string(),
            request_type: "stream".to_string(),
        },
    )
    .await;
    connection_handle.disarm();

    let mut http_queue_guard = Some(http_queue_guard);
    let stream = stream.inspect(move |response| {
        process_response_and_observe_metrics(
            response,
            &mut response_collector,
            &mut http_queue_guard,
        );
    });

    // Map each annotated NvVideosResponse to an MJPEG boundary chunk.
    // The backend yields one response per frame with the JPEG in data[0].b64_json.
    let mjpeg_stream = stream.filter_map(|annotated| async move {
        let ann = match annotated.ok() {
            Ok(a) => a,
            Err(e) => {
                tracing::error!("Video stream error: {e}");
                return None;
            }
        };
        let response = ann.data?;
        let frame = response.data.into_iter().next()?;
        let b64 = frame.b64_json?;
        let jpeg_bytes = match base64::prelude::BASE64_STANDARD.decode(&b64) {
            Ok(b) => b,
            Err(e) => {
                tracing::warn!("Failed to decode frame base64: {e}");
                return None;
            }
        };
        let header = format!(
            "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: {}\r\n\r\n",
            jpeg_bytes.len()
        );
        let mut chunk = Vec::with_capacity(header.len() + jpeg_bytes.len() + 2);
        chunk.extend_from_slice(header.as_bytes());
        chunk.extend_from_slice(&jpeg_bytes);
        chunk.extend_from_slice(b"\r\n");
        Some(Ok::<Bytes, std::convert::Infallible>(Bytes::from(chunk)))
    });

    // Arm the stream handle and monitor for client disconnects or context cancellation.
    // inflight.mark_ok() is deferred until the stream ends naturally. If the stream is
    // dropped early (client disconnect), the armed stream_handle signals the connection
    // monitor, which cancels the engine context.
    stream_handle.arm();
    let monitored_stream = async_stream::stream! {
        tokio::pin!(mjpeg_stream);
        loop {
            tokio::select! {
                frame = mjpeg_stream.next() => {
                    match frame {
                        Some(item) => yield item,
                        None => {
                            // Stream ended naturally: mark inflight OK and disarm the handle.
                            inflight.mark_ok();
                            stream_handle.disarm();
                            break;
                        }
                    }
                }
                _ = ctx.stopped() => {
                    tracing::trace!("Context stopped; breaking MJPEG stream");
                    inflight.mark_error(ErrorType::Cancelled);
                    break;
                }
            }
        }
    };

    axum::http::Response::builder()
        .status(axum::http::StatusCode::OK)
        .header(
            axum::http::header::CONTENT_TYPE,
            "multipart/x-mixed-replace; boundary=frame",
        )
        .body(Body::from_stream(monitored_stream))
        .map(|r| r.into_response())
        .map_err(|e| {
            // inflight is already owned by the monitored_stream which handles
            // mark_ok (stream end) and mark_error (cancellation).
            ErrorMessage::internal_server_error_with_details(
                "Failed to build MJPEG response",
                format!("{e}"),
            )
        })
}

/// Create an Axum [`Router`] for the OpenAI API Videos endpoint
/// If no path is provided, the default path is `/v1/videos`
///
/// Two routes are registered:
/// - `POST /v1/videos`        — non-streaming, returns a single JSON response
/// - `POST /v1/videos/stream` — MJPEG streaming via `multipart/x-mixed-replace`
pub fn videos_router(
    state: Arc<service_v2::State>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let path = path.unwrap_or("/v1/videos".to_string());
    let stream_path = format!("{}/stream", path);
    let doc = RouteDoc::new(axum::http::Method::POST, &path);
    let stream_doc = RouteDoc::new(axum::http::Method::POST, &stream_path);
    let router = Router::new()
        .route(&path, post(videos))
        .route(&stream_path, post(video_stream))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state(state);
    (vec![doc, stream_doc], router)
}

fn audio_content_type(format: &str) -> &'static str {
    match format {
        "mp3" => "audio/mpeg",
        "flac" => "audio/flac",
        "pcm" => "audio/pcm",
        "aac" => "audio/aac",
        "opus" => "audio/ogg; codecs=opus",
        _ => "audio/wav",
    }
}

fn decode_audio_chunks(response: &NvAudioSpeechResponse) -> Result<Vec<Bytes>, String> {
    response
        .data
        .iter()
        .map(|audio| {
            let encoded = audio
                .b64_json
                .as_deref()
                .ok_or_else(|| "Audio response did not contain base64 data".to_string())?;
            base64::engine::general_purpose::STANDARD
                .decode(encoded)
                .map(Bytes::from)
                .map_err(|e| format!("Failed to decode audio data: {e}"))
        })
        .collect()
}

async fn handler_audio_speech(
    State(state): State<Arc<service_v2::State>>,
    headers: HeaderMap,
    Json(mut request): Json<NvCreateAudioSpeechRequest>,
) -> Result<Response, ErrorResponse> {
    // return a 503 if the service is not ready
    // (per-model readiness check is deferred until after we resolve the
    // Option<String> model field; see below)
    check_ready(&state)?;

    let returns_audio_bytes = request.data_source.as_deref() != Some("url");
    let streams_audio_chunks = returns_audio_bytes
        && matches!(
            request.response_format.as_deref().unwrap_or("wav"),
            "pcm" | "wav"
        )
        && request.speed.is_none_or(|speed| speed == 1.0);
    let request_id = get_or_create_request_id(&headers);
    if streams_audio_chunks {
        // Advertise that this frontend can concatenate incremental worker
        // responses. Older frontends omit the signal, so new workers aggregate.
        // TODO(v1.7): Remove when v1.4 falls outside the N-2 window.
        request
            .nvext
            .get_or_insert_default()
            .frontend_accepts_audio_chunks = Some(true);
    }
    request.nest_passthrough();
    let mut request = context_from_headers(request, request_id, &headers)?;

    // model is optional in the request; fall back to a model that can actually
    // serve right now (complete worker set), not just any displayable one, so
    // an incomplete deployment doesn't get picked as the implicit default while
    // a ready model exists.
    let model = request.model.clone().unwrap_or_else(|| {
        state
            .manager()
            .serving_ready_display_names()
            .into_iter()
            .next()
            .unwrap_or_default()
    });
    // Per-model serving readiness gate (now that we have a resolved model
    // name string). Runs on the requested name so a 503 quotes back what the
    // caller asked for.
    check_model_serving_ready(&state, &model)?;

    // Audio registrations honor --served-model-name aliases, so resolve one to
    // its primary before it reaches routing, metrics, or the engine request.
    // Readiness is published per primary name, and every other alias-bearing
    // surface keeps the request model consistent with that name.
    let model = state.manager().resolve_canonical_name(&model);
    request.model = Some(model.clone());

    let context = request.context();
    let (mut connection_handle, stream_handle) = create_connection_monitor(
        context,
        Some(state.metrics_clone()),
        CancellationLabels {
            model: state.manager().metric_model_for(&model).to_string(),
            endpoint: Endpoint::Audios.to_string(),
            request_type: if streams_audio_chunks {
                "stream"
            } else {
                "unary"
            }
            .to_string(),
        },
    )
    .await;

    let response = tokio::spawn(
        audio_speech(
            state,
            request,
            model,
            returns_audio_bytes,
            streams_audio_chunks,
            stream_handle,
        )
        .in_current_span(),
    )
    .await
    .map_err(|e| {
        ErrorMessage::internal_server_error_with_details(
            "Failed to await audio speech task",
            format!("{e:?}"),
        )
    })?;

    connection_handle.disarm();
    response
}

async fn audio_speech(
    state: Arc<service_v2::State>,
    request: Context<NvCreateAudioSpeechRequest>,
    model: String,
    returns_audio_bytes: bool,
    streams_audio_chunks: bool,
    mut stream_handle: ConnectionHandle,
) -> Result<Response, ErrorResponse> {
    let request_id = request.id().to_string();
    let metric_model = state.manager().metric_model_for(&model).to_string();

    let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model);

    let engine = state
        .manager()
        .get_audios_engine(&model)
        .map_err(|e| ErrorMessage::from_model_error(&e))?;

    let mut inflight = state.metrics_clone().create_inflight_guard(
        &metric_model,
        Endpoint::Audios,
        streams_audio_chunks,
        &request_id,
    );

    let mut response_collector = state
        .metrics_clone()
        .create_response_collector(&metric_model);

    let ctx = request.context();
    inflight.mark_error(ErrorType::Cancelled);
    let stream = engine.generate(request).await.map_err(|e| {
        if super::metrics::request_was_rejected(e.as_ref()) {
            state
                .metrics_clone()
                .inc_rejection(&metric_model, super::metrics::Endpoint::Audios);
        }
        let err_response = ErrorMessage::from_anyhow(e, "Failed to generate audio");
        inflight.mark_error(extract_error_type_from_response(&err_response));
        err_response
    })?;

    let stream = check_for_backend_error(stream, None)
        .await
        .inspect_err(|error_response| {
            let error_type = match error_response.0 {
                // Worker-side InvalidArgument messages are not guaranteed to use
                // the "Validation:" prefix expected by the shared classifier.
                StatusCode::BAD_REQUEST => ErrorType::Validation,
                _ => extract_error_type_from_response(error_response),
            };
            inflight.mark_error(error_type);
        })?;

    let mut http_queue_guard = Some(http_queue_guard);
    let stream = stream.inspect(move |response| {
        process_response_and_observe_metrics(
            response,
            &mut response_collector,
            &mut http_queue_guard,
        );
    });

    if streams_audio_chunks {
        let mut stream = Box::pin(stream);
        let first_response = loop {
            let Some(annotated) = stream.next().await else {
                let err_response = ErrorMessage::internal_server_error(
                    "Audio stream ended without producing data",
                );
                inflight.mark_error(extract_error_type_from_response(&err_response));
                return Err(err_response);
            };
            let annotated = annotated.ok().map_err(|e| {
                let err_response = ErrorMessage::internal_server_error_with_details(
                    "Audio stream failed before producing data",
                    e.to_string(),
                );
                inflight.mark_error(extract_error_type_from_response(&err_response));
                err_response
            })?;
            let Some(response) = annotated.data else {
                continue;
            };
            if response.status == "failed" {
                inflight.mark_error(ErrorType::Validation);
                return Ok((StatusCode::BAD_REQUEST, Json(response)).into_response());
            }
            if !response.data.is_empty() {
                break response;
            }
        };

        let content_type = first_response
            .data
            .first()
            .map(|audio| audio_content_type(&audio.output_format))
            .unwrap_or("audio/wav");
        let first_chunks = decode_audio_chunks(&first_response).map_err(|e| {
            let err_response = ErrorMessage::internal_server_error_with_details(
                "Failed to decode audio stream",
                e,
            );
            inflight.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;
        if first_chunks.is_empty() {
            let err_response =
                ErrorMessage::internal_server_error("Audio response did not contain data");
            inflight.mark_error(extract_error_type_from_response(&err_response));
            return Err(err_response);
        }

        stream_handle.arm();

        let body_stream = async_stream::stream! {
            for chunk in first_chunks {
                yield Ok::<Bytes, std::io::Error>(chunk);
            }

            let stopped = ctx.stopped();
            tokio::pin!(stopped);
            loop {
                tokio::select! {
                    biased;
                    item = stream.next() => {
                        let Some(annotated) = item else {
                            inflight.mark_ok();
                            stream_handle.disarm();
                            break;
                        };
                        let annotated = match annotated.ok() {
                            Ok(annotated) => annotated,
                            Err(e) => {
                                inflight.mark_error(ErrorType::Internal);
                                stream_handle.disarm();
                                yield Err(std::io::Error::other(e.to_string()));
                                break;
                            }
                        };
                        let Some(response) = annotated.data else {
                            continue;
                        };
                        if response.status == "failed" {
                            inflight.mark_error(ErrorType::Internal);
                            stream_handle.disarm();
                            yield Err(std::io::Error::other(
                                response.error.unwrap_or_else(|| "Audio generation failed".to_string())
                            ));
                            break;
                        }
                        match decode_audio_chunks(&response) {
                            Ok(chunks) => {
                                for chunk in chunks {
                                    yield Ok(chunk);
                                }
                            }
                            Err(e) => {
                                inflight.mark_error(ErrorType::Internal);
                                stream_handle.disarm();
                                yield Err(std::io::Error::other(e));
                                break;
                            }
                        }
                    }
                    _ = &mut stopped => {
                        inflight.mark_error(ErrorType::Cancelled);
                        stream_handle.disarm();
                        break;
                    }
                }
            }
        };

        return Response::builder()
            .header("content-type", content_type)
            .body(Body::from_stream(body_stream))
            .map_err(|e| {
                ErrorMessage::internal_server_error_with_details(
                    "Failed to build audio response",
                    e.to_string(),
                )
            });
    }

    let response = NvAudioSpeechResponse::from_annotated_stream(stream)
        .await
        .map_err(|e| {
            let err_response =
                ErrorMessage::from_anyhow(anyhow::Error::new(e), "Failed to fold audio stream");
            inflight.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;

    // Check for failure before marking success
    if response.status == "failed" {
        // Without this the guard drops on its default and books this 400 as an
        // internal error.
        inflight.mark_error(ErrorType::Validation);
        return Ok((axum::http::StatusCode::BAD_REQUEST, Json(response)).into_response());
    }

    if returns_audio_bytes {
        let content_type = response
            .data
            .first()
            .map(|audio| audio_content_type(&audio.output_format))
            .unwrap_or("audio/wav");
        let chunks = decode_audio_chunks(&response).map_err(|e| {
            let err_response = ErrorMessage::internal_server_error_with_details(
                "Failed to decode audio response",
                e,
            );
            inflight.mark_error(extract_error_type_from_response(&err_response));
            err_response
        })?;
        if chunks.is_empty() {
            let err_response =
                ErrorMessage::internal_server_error("Audio response did not contain data");
            inflight.mark_error(extract_error_type_from_response(&err_response));
            return Err(err_response);
        }

        let content_length = chunks.iter().map(Bytes::len).sum();
        let mut audio_bytes = Vec::with_capacity(content_length);
        for chunk in chunks {
            audio_bytes.extend_from_slice(&chunk);
        }
        let response = Response::builder()
            .header("content-type", content_type)
            .header("content-length", content_length.to_string())
            .body(Body::from(audio_bytes))
            .map_err(|e| {
                let err_response = ErrorMessage::internal_server_error_with_details(
                    "Failed to build audio response",
                    e.to_string(),
                );
                inflight.mark_error(extract_error_type_from_response(&err_response));
                err_response
            })?;
        inflight.mark_ok();
        return Ok(response);
    }

    inflight.mark_ok();
    Ok(Json(response).into_response())
}

/// Create an Axum [`Router`] for the Audio Speech endpoint
/// Default path is `/v1/audio/speech`
pub fn audios_router(
    state: Arc<service_v2::State>,
    path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
    let path = path.unwrap_or("/v1/audio/speech".to_string());
    let doc = RouteDoc::new(axum::http::Method::POST, &path);
    let router = Router::new()
        .route(&path, post(handler_audio_speech))
        .layer(middleware::from_fn(smart_json_error_middleware))
        .layer(axum::extract::DefaultBodyLimit::max(get_body_limit()))
        .with_state(state);
    (vec![doc], router)
}

#[cfg(test)]
mod tests {

    use super::*;
    use crate::discovery::ModelManagerError;
    use crate::protocols::common::StopConditionsProvider;
    use crate::protocols::common::extensions::{AgentCompaction, NvExt};
    use crate::protocols::openai::chat_completions::NvCreateChatCompletionRequest;
    use crate::protocols::openai::common_ext::CommonExt;
    use crate::protocols::openai::completions::NvCreateCompletionRequest;
    use crate::protocols::openai::pooling::{PoolingData, PoolingUsage};
    use crate::protocols::openai::responses::NvCreateResponse;
    use dynamo_protocols::types::responses::{CreateResponse, Input, PromptConfig};
    use dynamo_protocols::types::{
        ChatCompletionRequestMessage, ChatCompletionRequestUserMessage,
        ChatCompletionRequestUserMessageContent, CreateChatCompletionRequest,
        CreateCompletionRequest, Prompt,
    };

    const BACKUP_ERROR_MESSAGE: &str = "Failed to generate completions";

    fn binary_pooling_response() -> NvCreatePoolingResponse {
        NvCreatePoolingResponse {
            id: "pool-request".to_string(),
            object: "list".to_string(),
            created: 123,
            model: "test-model".to_string(),
            data: vec![
                PoolingData {
                    index: 0,
                    object: "pooling".to_string(),
                    data: PoolingOutput::Base64(
                        base64::engine::general_purpose::STANDARD.encode([1, 2, 3, 4]),
                    ),
                    shape: Some(vec![2]),
                },
                PoolingData {
                    index: 1,
                    object: "pooling".to_string(),
                    data: PoolingOutput::Base64(
                        base64::engine::general_purpose::STANDARD.encode([5, 6, 7, 8]),
                    ),
                    shape: Some(vec![1, 2]),
                },
            ],
            usage: PoolingUsage {
                prompt_tokens: 7,
                total_tokens: 7,
                completion_tokens: 0,
            },
        }
    }

    #[tokio::test]
    async fn pooling_bytes_response_has_vllm_metadata_and_chunked_body() {
        let response = build_pooling_binary_response(
            binary_pooling_response(),
            true,
            PoolingEmbedDType::Float16,
            PoolingEndianness::Big,
        )
        .unwrap();

        assert_eq!(
            response.headers()[axum::http::header::CONTENT_TYPE],
            "application/octet-stream"
        );
        let metadata: serde_json::Value =
            serde_json::from_str(response.headers()["metadata"].to_str().unwrap()).unwrap();
        assert_eq!(
            metadata,
            serde_json::json!({
                "id": "pool-request",
                "created": 123,
                "model": "test-model",
                "data": [
                    {
                        "index": 0,
                        "embed_dtype": "float16",
                        "endianness": "big",
                        "start": 0,
                        "end": 4,
                        "shape": [2]
                    },
                    {
                        "index": 1,
                        "embed_dtype": "float16",
                        "endianness": "big",
                        "start": 4,
                        "end": 8,
                        "shape": [1, 2]
                    }
                ],
                "usage": {"prompt_tokens": 7, "total_tokens": 7}
            })
        );

        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
            .await
            .unwrap();
        assert_eq!(&body[..], &[1, 2, 3, 4, 5, 6, 7, 8]);
    }

    #[tokio::test]
    async fn pooling_bytes_only_response_omits_metadata() {
        let mut source = binary_pooling_response();
        for item in &mut source.data {
            item.shape = None;
        }
        let response = build_pooling_binary_response(
            source,
            false,
            PoolingEmbedDType::Float32,
            PoolingEndianness::Native,
        )
        .unwrap();

        assert!(response.headers().get("metadata").is_none());
        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
            .await
            .unwrap();
        assert_eq!(&body[..], &[1, 2, 3, 4, 5, 6, 7, 8]);
    }

    #[test]
    fn pooling_bytes_metadata_requires_tensor_shape() {
        let mut response = binary_pooling_response();
        response.data[0].shape = None;

        let error = build_pooling_binary_response(
            response,
            true,
            PoolingEmbedDType::Float32,
            PoolingEndianness::Native,
        )
        .unwrap_err();
        assert!(error.to_string().contains("missing its tensor shape"));
    }

    #[test]
    fn pooling_bytes_metadata_validates_tensor_size() {
        let mut response = binary_pooling_response();
        response.data[0].shape = Some(vec![3]);

        let error = build_pooling_binary_response(
            response,
            true,
            PoolingEmbedDType::Float16,
            PoolingEndianness::Native,
        )
        .unwrap_err();
        assert!(error.to_string().contains("requires 6"));
    }

    #[test]
    fn test_chat_completions_template_preserves_explicit_zero_temperature() {
        let template = RequestTemplate {
            model: "template-model".to_string(),
            temperature: 0.7,
            max_completion_tokens: 128,
        };
        let mut request = CreateChatCompletionRequest {
            temperature: Some(0.0),
            ..Default::default()
        };

        apply_chat_completions_request_template(&mut request, Some(&template));

        assert_eq!(request.temperature, Some(0.0));
        assert_eq!(request.model, "template-model");
        assert_eq!(request.max_completion_tokens, Some(128));

        request.temperature = None;
        apply_chat_completions_request_template(&mut request, Some(&template));
        assert_eq!(request.temperature, Some(0.7));
    }

    #[test]
    fn test_is_json_content_type() {
        assert!(is_json_content_type("application/json"));
        assert!(is_json_content_type("application/json; charset=utf-8"));
        assert!(is_json_content_type("Application/JSON"));
        assert!(is_json_content_type("application/vnd.dynamo+json"));
        assert!(!is_json_content_type("text/plain"));
        assert!(!is_json_content_type("application/json-patch"));
        assert!(!is_json_content_type("application"));
    }

    #[test]
    fn test_ensure_json_content_type_rejects_missing_or_non_json() {
        let headers = HeaderMap::new();
        let err = ensure_json_content_type(&headers).expect_err("missing content type should fail");
        assert_eq!(err.0, StatusCode::UNSUPPORTED_MEDIA_TYPE);

        let mut headers = HeaderMap::new();
        headers.insert(
            axum::http::header::CONTENT_TYPE,
            "text/plain".parse().unwrap(),
        );
        let err =
            ensure_json_content_type(&headers).expect_err("non-json content type should fail");
        assert_eq!(err.0, StatusCode::UNSUPPORTED_MEDIA_TYPE);
    }

    #[test]
    fn test_parse_chat_completion_request_escapes_control_chars_in_strings() {
        let body = b"{\"model\":\"test-model\",\"messages\":[{\"role\":\"user\",\"content\":\"log \x1b[33mPK\x03\x04\"}]}";

        let request: NvCreateChatCompletionRequest =
            parse_json_request("chat completions", body).expect("request should parse");

        let message = request
            .inner
            .messages
            .first()
            .expect("message should exist");
        let ChatCompletionRequestMessage::User(user_message) = message else {
            panic!("expected user message");
        };
        let ChatCompletionRequestUserMessageContent::Text(content) = &user_message.content else {
            panic!("expected text content");
        };
        assert_eq!(content, "log \u{1b}[33mPK\u{3}\u{4}");
    }

    #[test]
    fn test_parse_chat_completion_request_replaces_invalid_utf8_in_strings() {
        let body = b"{\"model\":\"test-model\",\"messages\":[{\"role\":\"user\",\"content\":\"raw \xff data\"}]}";

        let request: NvCreateChatCompletionRequest =
            parse_json_request("chat completions", body).expect("request should parse");

        let message = request
            .inner
            .messages
            .first()
            .expect("message should exist");
        let ChatCompletionRequestMessage::User(user_message) = message else {
            panic!("expected user message");
        };
        let ChatCompletionRequestUserMessageContent::Text(content) = &user_message.content else {
            panic!("expected text content");
        };
        assert_eq!(content, "raw \u{fffd} data");
    }

    #[test]
    fn test_parse_chat_completion_request_escapes_control_char_after_backslash() {
        let body = b"{\"model\":\"test-model\",\"messages\":[{\"role\":\"user\",\"content\":\"slash \\\nnext\"}]}";

        let request: NvCreateChatCompletionRequest =
            parse_json_request("chat completions", body).expect("request should parse");

        let message = request
            .inner
            .messages
            .first()
            .expect("message should exist");
        let ChatCompletionRequestMessage::User(user_message) = message else {
            panic!("expected user message");
        };
        let ChatCompletionRequestUserMessageContent::Text(content) = &user_message.content else {
            panic!("expected text content");
        };
        assert_eq!(content, "slash \\\nnext");
    }

    #[test]
    fn test_parse_chat_completion_request_keeps_schema_errors() {
        let body = br#"{"model":"test-model","messages":[{"role":"assistant","content":[{"type":"thinking","thinking":"working"}]}]}"#;

        let err =
            match parse_json_request::<NvCreateChatCompletionRequest>("chat completions", body) {
                Ok(_) => panic!("schema should still fail"),
                Err(err) => err,
            };

        assert_eq!(err.0, StatusCode::BAD_REQUEST);
        assert!(
            err.1
                .message
                .contains("ChatCompletionRequestAssistantMessageContent"),
            "unexpected error: {}",
            err.1.message
        );
    }

    #[test]
    fn test_parse_chat_completion_request_accepts_empty_image_url_with_uuid() {
        let body = br#"{"model":"test-model","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":""},"uuid":"image-42"}]}]}"#;

        let request: NvCreateChatCompletionRequest =
            parse_json_request("chat completions", body).expect("request should parse");
        let request = serde_json::to_value(request).expect("request should serialize");
        assert_eq!(request["messages"][0]["content"][0]["uuid"], "image-42");
        assert_eq!(
            request["messages"][0]["content"][0]["image_url"],
            serde_json::Value::Null
        );
    }

    #[test]
    fn test_parse_chat_completion_request_accepts_empty_uuid_url_after_tolerant_parse() {
        let body = b"{\"model\":\"test-model\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"raw \xff \x1b data\"},{\"type\":\"image_url\",\"image_url\":{\"url\":\"\"},\"uuid\":\"image-42\"}]}]}";

        let request: NvCreateChatCompletionRequest =
            parse_json_request("chat completions", body).expect("request should parse");
        let request = serde_json::to_value(request).expect("request should serialize");
        assert_eq!(
            request["messages"][0]["content"][0]["text"],
            "raw \u{fffd} \u{1b} data"
        );
        assert_eq!(
            request["messages"][0]["content"][1]["image_url"],
            serde_json::Value::Null
        );
        assert_eq!(request["messages"][0]["content"][1]["uuid"], "image-42");
    }

    #[test]
    fn test_parse_completion_request_escapes_control_chars_in_prompt() {
        let body =
            b"{\"model\":\"test-model\",\"prompt\":\"log \x1b[33mPK\x03\x04\",\"max_tokens\":1}";

        let request: NvCreateCompletionRequest =
            parse_json_request("completions", body).expect("request should parse");

        let Prompt::String(prompt) = &request.inner.prompt else {
            panic!("expected string prompt");
        };
        assert_eq!(prompt, "log \u{1b}[33mPK\u{3}\u{4}");
    }

    #[test]
    fn test_parse_completion_request_replaces_invalid_utf8_in_prompt() {
        let body = b"{\"model\":\"test-model\",\"prompt\":\"raw \xff data\",\"max_tokens\":1}";

        let request: NvCreateCompletionRequest =
            parse_json_request("completions", body).expect("request should parse");

        let Prompt::String(prompt) = &request.inner.prompt else {
            panic!("expected string prompt");
        };
        assert_eq!(prompt, "raw \u{fffd} data");
    }

    fn http_error_from_engine(code: u16) -> Result<(), anyhow::Error> {
        Err(HttpError {
            code,
            message: "custom error message".to_string(),
        })?
    }

    fn other_error_from_engine() -> Result<(), anyhow::Error> {
        Err(ModelManagerError::ModelNotFound("foo".to_string()))?
    }

    fn make_base_request() -> NvCreateResponse {
        NvCreateResponse {
            inner: CreateResponse {
                input: Input::Text("hello".into()),
                model: Some("test-model".into()),
                ..Default::default()
            },
            nvext: None,
            chat_template_args: None,
        }
    }

    #[test]
    fn responses_force_nonempty_request_requires_fallback_keep_alive() {
        let parsing_options = ParsingOptions::new(Some("qwen3_coder".into()), Some("qwen3".into()));
        let mut chat_template_args = HashMap::new();

        assert!(!request_stream_can_defer_all_output(&parsing_options, None));

        chat_template_args.insert(
            "force_nonempty_content".to_string(),
            serde_json::Value::Bool(false),
        );
        assert!(!request_stream_can_defer_all_output(
            &parsing_options,
            Some(&chat_template_args)
        ));

        chat_template_args.insert(
            "force_nonempty_content".to_string(),
            serde_json::Value::Bool(true),
        );
        assert!(request_stream_can_defer_all_output(
            &parsing_options,
            Some(&chat_template_args)
        ));

        let muse_tool_parser_only = ParsingOptions::new(Some("muse_glimmer".into()), None);
        assert!(request_stream_can_defer_all_output(
            &muse_tool_parser_only,
            Some(&chat_template_args)
        ));
    }

    #[test]
    fn test_responses_max_output_tokens_reaches_chat_budget_field() {
        let mut response_request = make_base_request();
        response_request.inner.max_output_tokens = Some(256);

        let unified_request: UnifiedRequest = response_request.try_into().unwrap();
        let chat_request = unified_request.into_inner();
        let stop_conditions = chat_request.extract_stop_conditions().unwrap();

        assert_eq!(chat_request.inner.max_completion_tokens, Some(256));
        assert_eq!(stop_conditions.max_tokens, Some(256));
    }

    #[test]
    fn test_openai_nvext_rejects_agent_context() {
        let err = serde_json::from_value::<NvExt>(serde_json::json!({
            "agent_context": {
                "session_id": "run-123"
            }
        }))
        .unwrap_err();

        assert!(err.to_string().contains("unknown field `agent_context`"));
    }

    #[test]
    fn test_copy_context_metadata_preserves_agent_context() {
        let mut source = Context::new(());
        source.insert(
            AGENT_CONTEXT_CONTEXT_KEY,
            AgentContext {
                session_id: "session-123".to_string(),
                parent_session_id: Some("parent-456".to_string()),
                session_final: Some(true),
                compaction: Some(AgentCompaction {
                    trigger: Some("automatic".to_string()),
                    ..Default::default()
                }),
                kv_hints: None,
                input_trigger: None,
            },
        );

        let mut target = Context::new(());
        copy_context_metadata(&source, &mut target);

        let agent_context = target
            .get::<AgentContext>(AGENT_CONTEXT_CONTEXT_KEY)
            .expect("agent context copied");
        assert_eq!(agent_context.session_id, "session-123");
        assert_eq!(
            agent_context.parent_session_id.as_deref(),
            Some("parent-456")
        );
        assert_eq!(agent_context.session_final, Some(true));
        assert_eq!(
            agent_context
                .compaction
                .as_ref()
                .and_then(|compaction| compaction.trigger.as_deref()),
            Some("automatic")
        );
    }

    #[test]
    fn test_context_from_headers_preserves_codex_compaction() {
        let mut headers = HeaderMap::new();
        headers.insert("thread-id", "codex-thread".parse().unwrap());
        headers.insert(
            "x-codex-turn-metadata",
            r#"{"request_kind":"compaction","compaction":{"trigger":"manual","reason":"user_requested","implementation":"local","phase":"summary_turn","strategy":"memento"}}"#
                .parse()
                .unwrap(),
        );

        let context = context_from_headers((), "request-1".to_string(), &headers).unwrap();
        let agent_context = context
            .get::<AgentContext>(AGENT_CONTEXT_CONTEXT_KEY)
            .expect("agent context attached");
        assert_eq!(agent_context.session_id, "codex-thread");
        assert_eq!(
            agent_context
                .compaction
                .as_ref()
                .and_then(|compaction| compaction.implementation.as_deref()),
            Some("local")
        );
    }

    #[test]
    fn test_context_from_headers_classifies_only_agent_requests() {
        let calls = std::cell::Cell::new(0);
        let classify = |_: &()| {
            calls.set(calls.get() + 1);
            Some(InputTrigger::Other)
        };

        context_from_headers_with_input_trigger(
            (),
            "request-1".to_string(),
            &HeaderMap::new(),
            classify,
        )
        .unwrap();
        assert_eq!(calls.get(), 0);

        let mut headers = HeaderMap::new();
        headers.insert("x-dynamo-session-id", "session-123".parse().unwrap());
        let source = context_from_headers_with_input_trigger(
            (),
            "request-2".to_string(),
            &headers,
            classify,
        )
        .unwrap();
        assert_eq!(calls.get(), 1);
        assert_eq!(
            source
                .get::<AgentContext>(AGENT_CONTEXT_CONTEXT_KEY)
                .unwrap()
                .input_trigger,
            Some(InputTrigger::Other)
        );
    }

    #[test]
    fn test_context_metadata_preserves_session_affinity() {
        let mut headers = HeaderMap::new();
        headers.insert("x-dynamo-session-id", "session-123".parse().unwrap());
        let source = context_from_headers((), "request-1".to_string(), &headers).unwrap();
        let affinity = source
            .get::<SessionAffinityId>(SESSION_AFFINITY_CONTEXT_KEY)
            .expect("session affinity attached");
        assert_eq!(affinity.as_str(), "session-123");

        let mut target = Context::new(());
        copy_context_metadata(&source, &mut target);
        let affinity = target
            .get::<SessionAffinityId>(SESSION_AFFINITY_CONTEXT_KEY)
            .expect("session affinity copied");
        assert_eq!(affinity.as_str(), "session-123");
    }

    #[test]
    fn test_http_error_response_from_anyhow() {
        let err = http_error_from_engine(400).unwrap_err();
        let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);
        assert_eq!(response.0, StatusCode::BAD_REQUEST);
        assert_eq!(response.1.message, "custom error message");
    }

    #[test]
    fn empty_pooling_cache_salt_is_rejected() {
        assert!(validate_pooling_cache_salt(None).is_ok());
        assert!(validate_pooling_cache_salt(Some("salt")).is_ok());

        let response = validate_pooling_cache_salt(Some("")).unwrap_err();
        assert_eq!(response.0, StatusCode::BAD_REQUEST);
        assert_eq!(
            response.1.message,
            "Parameter 'cache_salt' must be a non-empty string if provided."
        );
    }

    #[test]
    fn test_check_ready_rejects_draining_service() {
        let service = service_v2::HttpService::builder().build().unwrap();
        let state = service.state_clone();

        assert!(check_ready(&state).is_ok());

        state.start_draining();
        let response = check_ready(&state).unwrap_err();
        assert_eq!(response.0, StatusCode::SERVICE_UNAVAILABLE);
    }

    #[test]
    fn test_error_response_from_anyhow_out_of_range() {
        // Backend-supplied messages outside the 4xx range must NOT be
        // forwarded to the client — they may include internal paths. 503 keeps
        // its status, matching the streaming path
        // (`test_check_for_backend_error_with_503_preserves_status`); the rest
        // answer 500.
        for (code, expected_status) in [
            (399u16, 500u16),
            (500, 500),
            (501, 500),
            (503, 503),
            (507, 500),
        ] {
            let err = http_error_from_engine(code).unwrap_err();
            let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);
            assert_eq!(response.0.as_u16(), expected_status, "status for {code}");
            assert_eq!(response.1.code, expected_status, "body code for {code}");
            assert_eq!(response.1.message, "Internal server error");
            assert!(
                !response.1.message.contains("custom error message"),
                "client response must not include the backend-supplied HttpError message"
            );
        }
    }

    #[test]
    fn test_from_http_error_sanitizes_499_message() {
        // Backend may construct HttpError { code: 499, message: "..." }; that
        // message can carry context IDs / queue paths and must not leak.
        let err = HttpError {
            code: 499,
            message: "session abc-123 cancelled at /srv/queue.py:42".to_string(),
        };
        let response = ErrorMessage::from_http_error(err);
        assert_eq!(response.0.as_u16(), 499);
        assert_eq!(response.1.code, 499);
        assert_eq!(response.1.message, "Request cancelled");
        assert!(!response.1.message.contains("abc-123"));
        assert!(!response.1.message.contains("/srv/queue.py"));
    }

    #[test]
    fn test_from_http_error_preserves_529_overload_status() {
        // A deliberate load shed must stay distinguishable from an internal
        // error. The body is still sanitized: it may carry internal paths.
        let err = HttpError {
            code: 529,
            message: "site overloaded at /srv/pool.py:12".to_string(),
        };
        let response = ErrorMessage::from_http_error(err);
        assert_eq!(response.0.as_u16(), 529);
        assert_eq!(response.1.code, 529);
        assert_eq!(response.1.error_type, "Overloaded");
        assert!(
            !response.1.message.contains("/srv/pool.py"),
            "client response must not include the backend-supplied path"
        );
        assert!(
            !response.1.message.contains("site overloaded"),
            "client response must not include the backend-supplied HttpError message"
        );
    }

    #[test]
    fn test_from_http_error_529_classifies_as_overload_for_metrics() {
        // Observability half of the same bug: the metric recorded Internal
        // while the status was squashed, hiding load shedding.
        let response = ErrorMessage::from_http_error(HttpError {
            code: 529,
            message: "site overloaded".to_string(),
        });
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::Overload
        );
    }

    #[test]
    fn test_from_http_error_rejects_out_of_range_code() {
        // Codes outside the HTTP status space fall back to a sanitized 500.
        let err = HttpError {
            code: 1000,
            message: "bogus status from /srv/backend.py:7".to_string(),
        };
        let response = ErrorMessage::from_http_error(err);
        assert_eq!(response.0, StatusCode::INTERNAL_SERVER_ERROR);
        assert_eq!(response.1.code, 500);
        assert_eq!(response.1.message, "Internal server error");
        assert!(!response.1.message.contains("/srv/backend.py"));
    }

    /// Read the tunnelled backend status out of an error response body.
    fn tunnelled_backend_status(response: &ErrorResponse) -> Option<u64> {
        response.1.details.as_ref()?.get("backend_status")?.as_u64()
    }

    #[test]
    fn test_from_http_error_coerces_unlisted_5xx_and_tunnels_status() {
        // The client sees a generic 500 while the asserted status survives in
        // `details`. 507 is a WebDAV code no Dynamo component emits; 501 is
        // what the previous blanket pass-through forwarded verbatim.
        for code in [501u16, 502, 504, 507] {
            let response = ErrorMessage::from_http_error(HttpError {
                code,
                message: format!("engine failure {code} at /srv/pool.py:12"),
            });
            assert_eq!(
                response.0,
                StatusCode::INTERNAL_SERVER_ERROR,
                "status {code}"
            );
            assert_eq!(response.1.code, 500, "body code {code}");
            assert_eq!(response.1.message, "Internal server error");
            assert_eq!(
                tunnelled_backend_status(&response),
                Some(u64::from(code)),
                "asserted status must be tunnelled for {code}"
            );
            // `details` carries a number, never the backend's prose.
            let serialized = serde_json::to_string(&response.1.0).unwrap();
            assert!(
                !serialized.contains("/srv/pool.py"),
                "serialized body must not include the backend-supplied path for {code}"
            );
            assert!(
                !serialized.contains("engine failure"),
                "serialized body must not include the backend-supplied message for {code}"
            );
        }
    }

    #[test]
    fn test_from_http_error_preserves_retryable_5xx_without_tunnel() {
        // These two survive on the status line, so nothing lands in `details`.
        for status in [StatusCode::SERVICE_UNAVAILABLE, overload_status_code()] {
            let response = ErrorMessage::from_http_error(HttpError {
                code: status.as_u16(),
                message: "shedding load at /srv/pool.py:12".to_string(),
            });
            assert_eq!(response.0, status);
            assert_eq!(response.1.code, status.as_u16());
            assert_eq!(response.1.message, "Internal server error");
            assert_eq!(
                tunnelled_backend_status(&response),
                None,
                "a preserved status must not also be tunnelled"
            );
        }
    }

    #[test]
    fn test_from_http_error_forwards_4xx_verbatim() {
        // The 5xx allowlist leaves 4xx alone: the backend's own description is
        // what the caller needs (e.g. the in-tree 415 from image loading).
        let response = ErrorMessage::from_http_error(HttpError {
            code: 415,
            message: "Unsupported Media Type: image/tiff".to_string(),
        });
        assert_eq!(response.0, StatusCode::UNSUPPORTED_MEDIA_TYPE);
        assert_eq!(response.1.code, 415);
        assert_eq!(response.1.message, "Unsupported Media Type: image/tiff");
        assert_eq!(tunnelled_backend_status(&response), None);
    }

    #[test]
    fn test_other_error_response_from_anyhow() {
        // Non-HttpError anyhow chains must NOT be exposed to the client; only
        // the static backup message should appear in the response.
        let err = other_error_from_engine().unwrap_err();
        let leaked_chain = format!("{err:#}");
        let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);
        assert_eq!(response.0, StatusCode::INTERNAL_SERVER_ERROR);
        assert_eq!(response.1.message, BACKUP_ERROR_MESSAGE);
        assert!(
            !response.1.message.contains(&leaked_chain),
            "client response must not contain the anyhow error chain"
        );
    }

    #[test]
    fn overload_errors_preserve_the_http_529_contract() {
        use dynamo_runtime::error::{DynamoError, ErrorType};
        use dynamo_runtime::pipeline::error::PipelineError;

        for (error_type, message) in [
            (
                ErrorType::ResourceExhausted,
                "All workers are busy, please retry later",
            ),
            (
                ErrorType::WorkerOverloaded,
                "Selected worker is overloaded, please retry later",
            ),
        ] {
            let cause = PipelineError::ServiceOverloaded(message.to_string());
            let err: anyhow::Error = DynamoError::builder()
                .error_type(error_type)
                .message(message)
                .cause(cause)
                .build()
                .into();
            let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);
            assert_eq!(response.0.as_u16(), 529);
            assert_eq!(response.1.code, 529);
            assert_eq!(response.1.error_type, "Overloaded");
            assert_eq!(response.1.message, "Service temporarily overloaded");
            assert!(
                !response.1.message.contains(message),
                "client response must not include the underlying engine message"
            );
        }
    }

    #[test]
    fn backend_overload_reports_overload_status_not_worker_status() {
        use dynamo_runtime::error::{DynamoError, ErrorType};

        // Production path: a vLLM worker rejects on its own slot limit and puts
        // 503 in the payload. The chain says ResourceExhausted, so the client
        // must see the overload status rather than a generic outage.
        let event: Annotated<NvCreateChatCompletionStreamResponse> = Annotated {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: None,
            error: Some(
                DynamoError::builder()
                    .error_type(ErrorType::ResourceExhausted)
                    .message(
                        r#"{"message":"Worker local total request limit reached (32/32)","code":503}"#,
                    )
                    .build(),
            ),
        };

        let backend_error =
            extract_backend_error_if_present(&event).expect("error event should be extracted");
        assert_eq!(backend_error.status, overload_status_code());
        assert_eq!(backend_error.status.as_u16(), 529);
        assert!(backend_error.message.contains("request limit reached"));
        // Carried, not re-derived from the status — this is what keeps the
        // rendering identical to the admission path at any configured status.
        assert!(matches!(
            backend_error.sanitized,
            Some(SanitizedError::Overloaded)
        ));
    }

    #[test]
    fn backend_non_overload_status_is_still_preserved() {
        use dynamo_runtime::error::{DynamoError, ErrorType};

        // The override is scoped to capacity rejections; a genuine backend
        // failure must keep the status the worker chose.
        let event: Annotated<NvCreateChatCompletionStreamResponse> = Annotated {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: None,
            error: Some(
                DynamoError::builder()
                    .error_type(ErrorType::Unknown)
                    .message(r#"{"message":"engine crashed","code":503}"#)
                    .build(),
            ),
        };

        let backend_error =
            extract_backend_error_if_present(&event).expect("error event should be extracted");
        assert_eq!(backend_error.status, StatusCode::SERVICE_UNAVAILABLE);
        assert!(backend_error.sanitized.is_none());
    }

    #[test]
    fn backend_overload_is_sanitized_at_a_non_5xx_overload_status() {
        // DYN_HTTP_OVERLOAD_STATUS_CODE accepts 200-999. Deriving the category
        // from the status alone sends a 4xx overload down the forward-verbatim
        // path, leaking the worker's internal text; a 2xx/3xx one down the
        // coerce-to-500 path, dropping the configured status. The carried
        // category avoids both, so the worker path renders exactly like the
        // admission path at every configured value.
        let response = backend_error_response(BackendErrorInfo {
            message: "Worker local total request limit reached (32/32)".to_string(),
            status: StatusCode::TOO_MANY_REQUESTS,
            sanitized: Some(SanitizedError::Overloaded),
        });

        assert_eq!(response.0, overload_status_code());
        assert_eq!(response.1.code, overload_status_code().as_u16());
        assert_eq!(response.1.message, SanitizedError::Overloaded.to_string());
        assert!(!response.1.message.contains("32/32"));
    }

    #[test]
    fn python_worker_503_reaches_the_frontend_as_backend_unknown() {
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType};

        // Exactly what map_python_exception (bindings/python/rust/engine.rs) and
        // py_err_to_dynamo (backend.rs) build for a Python exception carrying
        // `.code = 503`: 503 is outside 400..500, so the type is Backend(Unknown)
        // and the message is the JSON envelope. No cause is attached.
        let event: Annotated<NvCreateChatCompletionStreamResponse> = Annotated {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: None,
            error: Some(
                DynamoError::builder()
                    .error_type(ErrorType::Backend(BackendError::Unknown))
                    .message(
                        r#"{"message":"Worker local total request limit reached (32/32)","code":503}"#,
                    )
                    .build(),
            ),
        };

        // request_was_rejected keys on ErrorType::ResourceExhausted, which this
        // shape never carries, so the overload override does not engage.
        assert!(!super::super::metrics::request_was_rejected(
            event.error.as_ref().expect("error is set")
        ));

        let backend_error =
            extract_backend_error_if_present(&event).expect("error event should be extracted");
        assert_eq!(backend_error.status, StatusCode::SERVICE_UNAVAILABLE);
    }

    #[test]
    fn unavailable_error_response_from_anyhow() {
        use dynamo_runtime::error::{DynamoError, ErrorType};

        let err: anyhow::Error = DynamoError::builder()
            .error_type(ErrorType::Unavailable)
            .message("No workers available for endpoint test/worker/generate")
            .build()
            .into();
        let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);

        assert_eq!(response.0, StatusCode::SERVICE_UNAVAILABLE);
        assert_eq!(response.1.code, StatusCode::SERVICE_UNAVAILABLE.as_u16());
        assert_eq!(response.1.message, "Service temporarily unavailable");
    }

    #[test]
    fn queue_rejection_maps_to_structured_http_529() {
        use dynamo_kv_router::scheduling::{QueueLimitKind, QueueRejection};

        let rejection = QueueRejection {
            policy_class: "latency".to_string(),
            limit_kind: QueueLimitKind::CachedTokens,
            current: 2048,
            limit: 1024,
        };
        let response =
            ErrorMessage::from_anyhow(anyhow::Error::new(rejection), BACKUP_ERROR_MESSAGE);

        assert_eq!(response.0.as_u16(), 529);
        assert_eq!(response.1.code, 529);
        assert_eq!(response.1.error_type, "Overloaded");
        assert_eq!(
            response.1.details.as_deref(),
            Some(&serde_json::json!({
                "policy_class": "latency",
                "limit_kind": "cached_tokens",
                "current": 2048,
                "limit": 1024,
            }))
        );
    }

    #[test]
    fn test_nested_invalid_argument_response_from_anyhow() {
        use dynamo_runtime::error::{DynamoError, ErrorType};

        #[derive(Debug)]
        struct WrappedError {
            source: DynamoError,
        }

        impl std::fmt::Display for WrappedError {
            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                write!(f, "outer routing failure")
            }
        }

        impl std::error::Error for WrappedError {
            fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
                Some(&self.source)
            }
        }

        let source = DynamoError::builder()
            .error_type(ErrorType::InvalidArgument)
            .message(
                "Request payload is too large for this deployment. Reduce the input size or metadata size and retry.",
            )
            .build();
        let err: anyhow::Error = WrappedError { source }.into();

        let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);

        assert_eq!(response.0, StatusCode::BAD_REQUEST);
        assert_eq!(response.1.code, StatusCode::BAD_REQUEST.as_u16());
        assert!(response.1.message.contains("Request payload is too large"));
        assert!(!response.1.message.contains("NATS"));
        assert!(!response.1.message.contains("payload_bytes"));
    }

    #[test]
    fn test_backend_invalid_argument_surfaces_as_400() {
        // `Backend(InvalidArgument)` is what `py_err_to_dynamo` produces
        // for Python `ValueError` / `TypeError` raised inside an engine's
        // `generate()` — must map to 400, not 500.
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType};

        let err: anyhow::Error = DynamoError::builder()
            .error_type(ErrorType::Backend(BackendError::InvalidArgument))
            .message("Dynamo's SGLang backend does not currently support logprobs >= 1")
            .build()
            .into();

        let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);

        assert_eq!(response.0, StatusCode::BAD_REQUEST);
        assert_eq!(response.1.code, StatusCode::BAD_REQUEST.as_u16());
        assert!(response.1.message.contains("does not currently support"));
    }

    /// A worker that refuses a request before the response stream opens must
    /// reach the client as 400, not 500.
    #[test]
    fn test_pre_stream_refusal_surfaces_as_400() {
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType};
        use dynamo_runtime::pipeline::network::StreamPrologueError;
        use dynamo_runtime::pipeline::network::egress::addressed_router::testing::pre_stream_failure_error;

        let prologue_error = StreamPrologueError::new(
            "Generate Error: multimodal input is not supported by this backend",
            DynamoError::builder()
                .error_type(ErrorType::Backend(BackendError::InvalidArgument))
                .message("multimodal input is not supported by this backend")
                .build(),
        );

        let err: anyhow::Error = pre_stream_failure_error(prologue_error).into();
        let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);

        assert_eq!(response.0, StatusCode::BAD_REQUEST);
        assert_eq!(response.1.code, StatusCode::BAD_REQUEST.as_u16());
        assert!(
            response
                .1
                .message
                .contains("multimodal input is not supported"),
            "the client should see the worker's reason, got: {}",
            response.1.message
        );
    }

    /// `py_err_to_dynamo` wraps an HTTP-like Python exception's text in a
    /// `{"message":..,"code":..}` envelope, so a refusal from a Python worker
    /// carries JSON, not prose. The client must be shown the reason, never the
    /// envelope -- the in-stream path already unwraps it, and the two must agree.
    #[test]
    fn test_pre_stream_refusal_unwraps_the_python_error_envelope() {
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType};
        use dynamo_runtime::pipeline::network::StreamPrologueError;
        use dynamo_runtime::pipeline::network::egress::addressed_router::testing::pre_stream_failure_error;

        let refuse = |message: &str| {
            let prologue = StreamPrologueError::new(
                format!("Generate Error: {message}"),
                DynamoError::builder()
                    .error_type(ErrorType::Backend(BackendError::InvalidArgument))
                    .message(message)
                    .build(),
            );
            ErrorMessage::from_anyhow(
                pre_stream_failure_error(prologue).into(),
                BACKUP_ERROR_MESSAGE,
            )
        };

        let response = refuse(
            &serde_json::json!({"message": "multimodal input is not supported", "code": 400})
                .to_string(),
        );
        assert_eq!(response.0, StatusCode::BAD_REQUEST);
        assert_eq!(response.1.message, "multimodal input is not supported");

        // An explicit client-error status inside the envelope is honoured.
        let response = refuse(
            &serde_json::json!({"message": "unsupported media type", "code": 415}).to_string(),
        );
        assert_eq!(response.0, StatusCode::UNSUPPORTED_MEDIA_TYPE);
        assert_eq!(response.1.message, "unsupported media type");

        // A 5xx inside the envelope does not escape through the 4xx arm.
        let response = refuse(&serde_json::json!({"message": "boom", "code": 500}).to_string());
        assert_eq!(response.0, StatusCode::BAD_REQUEST);
    }

    /// A worker can report a cancellation as an HTTP-like 499, and its own text
    /// may name a context id or an internal path. The status survives, the text
    /// does not: this arm owes the client the same sanitized body every other
    /// HTTP path produces for a cancellation.
    #[test]
    fn test_pre_stream_refusal_sanitizes_the_envelope_cancellation() {
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType as DynErrorType};
        use dynamo_runtime::pipeline::network::StreamPrologueError;
        use dynamo_runtime::pipeline::network::egress::addressed_router::testing::pre_stream_failure_error;

        let envelope = serde_json::json!({
            "message": "Context id abc-123 stopped at /opt/dynamo/worker.py:42",
            "code": 499,
        })
        .to_string();
        let prologue = StreamPrologueError::new(
            format!("Generate Error: {envelope}"),
            DynamoError::builder()
                .error_type(DynErrorType::Backend(BackendError::InvalidArgument))
                .message(envelope)
                .build(),
        );

        let response = ErrorMessage::from_anyhow(
            pre_stream_failure_error(prologue).into(),
            BACKUP_ERROR_MESSAGE,
        );

        assert_eq!(response.0, StatusCode::from_u16(499).unwrap());
        assert_eq!(response.1.message, SanitizedError::Cancelled.to_string());
        assert!(
            !response.1.message.contains("abc-123") && !response.1.message.contains("/opt/dynamo"),
            "the worker's own cancellation text must not reach the client, got: {}",
            response.1.message
        );
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::Cancelled
        );
    }

    /// A status the envelope preserved is what the metric is counted from. A
    /// backend rate limit is an overload, not a validation failure; a plain 400
    /// keeps the validation override, because a worker's refusal text carries no
    /// `Validation:` prefix and would otherwise be counted as internal.
    #[test]
    fn test_pre_stream_refusal_classifies_metrics_from_the_preserved_status() {
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType as DynErrorType};
        use dynamo_runtime::pipeline::network::StreamPrologueError;
        use dynamo_runtime::pipeline::network::egress::addressed_router::testing::pre_stream_failure_error;

        let refuse = |message: String| {
            let prologue = StreamPrologueError::new(
                format!("Generate Error: {message}"),
                DynamoError::builder()
                    .error_type(DynErrorType::Backend(BackendError::InvalidArgument))
                    .message(message)
                    .build(),
            );
            ErrorMessage::from_anyhow(
                pre_stream_failure_error(prologue).into(),
                BACKUP_ERROR_MESSAGE,
            )
        };

        let rate_limited =
            refuse(serde_json::json!({"message": "too many requests", "code": 429}).to_string());
        assert_eq!(rate_limited.0, StatusCode::TOO_MANY_REQUESTS);
        assert_eq!(
            extract_error_type_from_response(&rate_limited),
            ErrorType::Overload
        );

        let refused = refuse("multimodal input is not supported by this backend".to_string());
        assert_eq!(refused.0, StatusCode::BAD_REQUEST);
        assert_eq!(
            extract_error_type_from_response(&refused),
            ErrorType::Validation
        );
    }

    /// Negative control for `test_pre_stream_refusal_surfaces_as_400`: a genuine
    /// pre-stream connect failure must still be 500. Only a request-level
    /// refusal earns a 4xx.
    #[test]
    fn test_pre_stream_connect_failure_still_surfaces_as_500() {
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType};
        use dynamo_runtime::pipeline::network::StreamPrologueError;
        use dynamo_runtime::pipeline::network::egress::addressed_router::testing::pre_stream_failure_error;

        // The worker died rather than refused: typed, but not a request problem.
        let engine_shutdown = StreamPrologueError::new(
            "Generate Error: engine shut down",
            DynamoError::builder()
                .error_type(ErrorType::Backend(BackendError::EngineShutdown))
                .message("engine shut down")
                .build(),
        );
        let response = ErrorMessage::from_anyhow(
            pre_stream_failure_error(engine_shutdown).into(),
            BACKUP_ERROR_MESSAGE,
        );
        assert_eq!(response.0, StatusCode::INTERNAL_SERVER_ERROR);

        // An older worker sends no typed error at all: also still 500.
        let untyped = StreamPrologueError::from_message("Generate Error: could not reach worker");
        let response = ErrorMessage::from_anyhow(
            pre_stream_failure_error(untyped).into(),
            BACKUP_ERROR_MESSAGE,
        );
        assert_eq!(response.0, StatusCode::INTERNAL_SERVER_ERROR);
    }

    #[test]
    fn test_cancelled_error_response_from_anyhow() {
        use dynamo_runtime::error::{DynamoError, ErrorType};

        let err: anyhow::Error = DynamoError::builder()
            .error_type(ErrorType::Cancelled)
            .message("Context id abc-123 is stopped or killed")
            .build()
            .into();
        let response = ErrorMessage::from_anyhow(err, BACKUP_ERROR_MESSAGE);
        assert_eq!(
            response.0.as_u16(),
            499,
            "Cancelled errors should return HTTP 499"
        );
        assert_eq!(response.1.code, 499);
        assert_eq!(response.1.error_type, "Client Closed Request");
        // The client gets a static message; the backend detail (context id,
        // cancellation internals) must not leak into the 499 body.
        assert_eq!(response.1.message, "Request cancelled");
        assert!(!response.1.message.contains("abc-123"));
        assert!(!response.1.message.contains("stopped or killed"));
    }

    #[test]
    fn test_cancelled_error_metrics_classification() {
        // HTTP 499 should be classified as Cancelled for metrics
        let error_type =
            classify_error_for_metrics(StatusCode::from_u16(499).unwrap(), "cancelled request");
        assert_eq!(
            error_type,
            ErrorType::Cancelled,
            "HTTP 499 should map to ErrorType::Cancelled in metrics"
        );
    }

    #[test]
    fn test_validate_unsupported_fields_accepts_clean_request() {
        let request = make_base_request();
        let result = validate_response_unsupported_fields(&request);
        assert!(result.is_none());
    }

    #[test]
    fn test_validate_unsupported_fields_accepts_parallel_tool_calls() {
        let mut request = make_base_request();
        request.inner.parallel_tool_calls = Some(true);
        let result = validate_response_unsupported_fields(&request);
        assert!(result.is_none(), "parallel_tool_calls should be supported");
    }

    #[test]
    fn test_validate_unsupported_fields_accepts_store() {
        let mut request = make_base_request();
        request.inner.store = Some(true);
        let result = validate_response_unsupported_fields(&request);
        assert!(
            result.is_none(),
            "store should be supported for audit opt-in"
        );
    }

    #[tokio::test]
    async fn test_validate_unsupported_fields_rejects_rl_nvext_fields() {
        for field in ["completion_token_ids", "prompt_logprobs"] {
            for stream in [false, true] {
                let mut request = make_base_request();
                request.inner.stream = Some(stream);
                request.nvext = Some(
                    NvExt::builder()
                        .extra_fields(vec![field.to_string()])
                        .build()
                        .unwrap(),
                );

                let response = validate_response_unsupported_fields(&request)
                    .expect("RL nvext response field should be rejected")
                    .into_response();
                assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED);

                let body = axum::body::to_bytes(response.into_body(), get_body_limit())
                    .await
                    .unwrap();
                let error: ErrorMessage = serde_json::from_slice(&body).unwrap();
                assert_eq!(
                    error.message,
                    format!(
                        "{VALIDATION_PREFIX}`nvext.extra_fields=[\"{field}\"]` is not supported by the Responses API."
                    )
                );
            }
        }
    }

    #[test]
    fn test_validate_unsupported_fields_rejects_mixed_nvext_fields() {
        let mut request = make_base_request();
        request.nvext = Some(
            NvExt::builder()
                .extra_fields(vec![
                    "timing".to_string(),
                    "completion_token_ids".to_string(),
                ])
                .build()
                .unwrap(),
        );

        assert!(validate_response_unsupported_fields(&request).is_some());
    }

    #[test]
    fn test_validate_unsupported_fields_accepts_supported_nvext_fields() {
        let mut request = make_base_request();
        request.nvext = Some(
            NvExt::builder()
                .extra_fields(vec!["timing".to_string(), "worker_id".to_string()])
                .build()
                .unwrap(),
        );

        assert!(validate_response_unsupported_fields(&request).is_none());
    }

    #[test]
    fn test_validate_responses_fields_accepts_clean_request() {
        assert!(validate_responses_fields(&make_base_request()).is_ok());
    }

    #[test]
    fn test_validate_responses_fields_rejects_zero_max_output_tokens() {
        let mut request = make_base_request();
        request.inner.max_output_tokens = Some(0);

        let (code, body) =
            validate_responses_fields(&request).expect_err("max_output_tokens: 0 must be rejected");
        assert_eq!(code, StatusCode::BAD_REQUEST);
        assert_eq!(
            body.message,
            format!("{VALIDATION_PREFIX}Max tokens must be greater than 0, got 0")
        );
    }

    #[test]
    fn test_validate_responses_fields_rejects_zero_top_p() {
        let mut request = make_base_request();
        request.inner.top_p = Some(0.0);

        let (code, body) =
            validate_responses_fields(&request).expect_err("top_p: 0 must be rejected");
        assert_eq!(code, StatusCode::BAD_REQUEST);
        assert_eq!(
            body.message,
            format!("{VALIDATION_PREFIX}Top_p must be between 0 and 1, got 0")
        );
    }

    #[test]
    fn test_validate_responses_fields_rejects_non_object_json_schema() {
        use dynamo_protocols::types::ResponseFormatJsonSchema;
        use dynamo_protocols::types::responses::{
            ResponseTextParam, TextResponseFormatConfiguration,
        };

        let mut request = make_base_request();
        request.inner.text = Some(ResponseTextParam {
            format: TextResponseFormatConfiguration::JsonSchema(ResponseFormatJsonSchema {
                name: "city".into(),
                description: None,
                schema: serde_json::json!(42), // Invalid: not an object
                strict: None,
            }),
            verbosity: None,
        });

        let (code, body) = validate_responses_fields(&request)
            .expect_err("non-object json_schema must be rejected");
        assert_eq!(code, StatusCode::BAD_REQUEST);
        assert!(body.message.contains("must be a JSON object"));
    }

    #[test]
    fn test_validate_responses_fields_accepts_object_json_schema() {
        use dynamo_protocols::types::ResponseFormatJsonSchema;
        use dynamo_protocols::types::responses::{
            ResponseTextParam, TextResponseFormatConfiguration,
        };

        let mut request = make_base_request();
        request.inner.text = Some(ResponseTextParam {
            format: TextResponseFormatConfiguration::JsonSchema(ResponseFormatJsonSchema {
                name: "city".into(),
                description: None,
                schema: serde_json::json!({"type": "object"}),
                strict: None,
            }),
            verbosity: None,
        });

        assert!(validate_responses_fields(&request).is_ok());
    }

    #[test]
    fn test_validate_unsupported_fields_detects_flags() {
        #[allow(clippy::type_complexity)]
        let unsupported_cases: Vec<(&str, Box<dyn FnOnce(&mut CreateResponse)>)> = vec![
            ("background", Box::new(|r| r.background = Some(true))),
            (
                "previous_response_id",
                Box::new(|r| r.previous_response_id = Some("prev-id".into())),
            ),
            (
                "prompt",
                Box::new(|r| {
                    r.prompt = Some(PromptConfig {
                        id: "template-id".into(),
                        version: None,
                        variables: None,
                    })
                }),
            ),
            ("max_tool_calls", Box::new(|r| r.max_tool_calls = Some(5))),
        ];

        for (field, set_field) in unsupported_cases {
            let mut req = make_base_request();
            (set_field)(&mut req.inner);
            let result = validate_response_unsupported_fields(&req);
            assert!(result.is_some(), "Expected rejection for `{field}`");
        }
    }

    /// Pass-through metadata fields (`prompt_cache_key`,
    /// `prompt_cache_retention`, `safety_identifier`) are accepted at the
    /// validation layer; the response serializer echoes them back so the
    /// caller can confirm receipt. Codex sends `prompt_cache_key` on every
    /// request — rejecting it broke `codex exec` end-to-end.
    #[test]
    fn test_validate_unsupported_fields_accepts_passthrough_metadata() {
        #[allow(clippy::type_complexity)]
        let passthrough_cases: Vec<(&str, Box<dyn FnOnce(&mut CreateResponse)>)> = vec![
            (
                "prompt_cache_key",
                Box::new(|r| r.prompt_cache_key = Some("ck-1".into())),
            ),
            (
                "prompt_cache_retention",
                Box::new(|r| {
                    r.prompt_cache_retention =
                        Some(dynamo_protocols::types::responses::PromptCacheRetention::InMemory)
                }),
            ),
            (
                "safety_identifier",
                Box::new(|r| r.safety_identifier = Some("user-hash".into())),
            ),
        ];

        for (field, set_field) in passthrough_cases {
            let mut req = make_base_request();
            (set_field)(&mut req.inner);
            let result = validate_response_unsupported_fields(&req);
            assert!(
                result.is_none(),
                "Expected `{field}` to be accepted as pass-through metadata"
            );
        }
    }

    #[test]
    fn test_validate_chat_completion_required_fields_empty_messages() {
        let request = NvCreateChatCompletionRequest {
            inner: CreateChatCompletionRequest {
                model: "test-model".to_string(),
                messages: vec![],
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            chat_template_args: None,
            thinking: None,
            media_io_kwargs: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_chat_completion_required_fields(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!(
                    "{VALIDATION_PREFIX}The 'messages' field cannot be empty. At least one message is required."
                )
            );
        }
    }

    #[test]
    fn test_validate_chat_completion_required_fields_with_messages() {
        let request = NvCreateChatCompletionRequest {
            inner: CreateChatCompletionRequest {
                model: "test-model".to_string(),
                messages: vec![ChatCompletionRequestMessage::User(
                    ChatCompletionRequestUserMessage {
                        content: ChatCompletionRequestUserMessageContent::Text("Hello".to_string()),
                        name: None,
                    },
                )],
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            chat_template_args: None,
            thinking: None,
            media_io_kwargs: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_chat_completion_required_fields(&request);
        assert!(result.is_ok());
    }

    #[test]
    fn test_normalize_chat_reasoning_template_args_error_response() {
        let mut request: NvCreateChatCompletionRequest =
            serde_json::from_value(serde_json::json!({
                "model": "test-model",
                "messages": [{"role": "user", "content": "Hello"}],
                "thinking": {"type": "auto"}
            }))
            .unwrap();

        let result = normalize_chat_reasoning_template_args(&mut request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!(
                    "{VALIDATION_PREFIX}`thinking.type` must be `enabled`, `disabled`, or `adaptive`"
                )
            );
        }
    }

    #[test]
    // Test for all Bad Requests Example for Chat Completion
    // 1. Echo:  Should be a boolean : Not Done
    // 2. Frequency Penalty: Should be a float between -2.0 and 2.0 : Done
    // 3. logprobs: Done
    // 4. Model Format: Should be a string : Not Done
    // 5. Prompt or Messages Validation
    // 6. Max Tokens: Should be a positive integer
    // 7. Presence Penalty: Should be a float between -2.0 and 2.0 : Done
    // 8. Stop : Should be a string or an array of strings : Not Done
    // 9. Invalid or Out of range temperature: Done
    // 10.Invalid or out of range top_p: Done
    // 11. Repetition Penalty: Should be a float between 0.0 and 2.0 : Done
    // 12. Logprobs: Should be a positive integer between 0 and 5 : Done
    // invalid or non existing user : Only empty string is not allowed validation is there. How can we check non-extisting user ?
    // Unknown fields : Done (rejected via extra_fields catch-all)
    // guided_whitespace_pattern null or invalid : Not Done
    // "response_format": { "type": "invalid_format" } : Not Done
    // "logit_bias": { "invalid_token": "not_a_number" }, : Partial Validation is already there
    fn test_bad_base_request_for_completion() {
        // Frequency Penalty: Should be a float between -2.0 and 2.0
        let request = NvCreateCompletionRequest {
            inner: CreateCompletionRequest {
                model: "test-model".to_string(),
                prompt: "Hello".into(),
                frequency_penalty: Some(-3.0),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            metadata: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };

        let result = validate_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Frequency penalty must be between -2 and 2, got -3")
            );
        }

        // Presence Penalty: Should be a float between -2.0 and 2.0
        let request = NvCreateCompletionRequest {
            inner: CreateCompletionRequest {
                model: "test-model".to_string(),
                prompt: "Hello".into(),
                presence_penalty: Some(-3.0),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            metadata: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Presence penalty must be between -2 and 2, got -3")
            );
        }

        // Temperature: Should be a float between 0.0 and 2.0
        let request = NvCreateCompletionRequest {
            inner: CreateCompletionRequest {
                model: "test-model".to_string(),
                prompt: "Hello".into(),
                temperature: Some(-3.0),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            metadata: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Temperature must be between 0 and 2, got -3")
            );
        }

        // Top P: Should be a float between 0.0 and 1.0
        let request = NvCreateCompletionRequest {
            inner: CreateCompletionRequest {
                model: "test-model".to_string(),
                prompt: "Hello".into(),
                top_p: Some(-3.0),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            metadata: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Top_p must be between 0 and 1, got -3")
            );
        }

        // Repetition Penalty: Should be a float between 0.0 and 2.0
        let request = NvCreateCompletionRequest {
            inner: CreateCompletionRequest {
                model: "test-model".to_string(),
                prompt: "Hello".into(),
                ..Default::default()
            },
            common: CommonExt::builder()
                .repetition_penalty(-3.0)
                .build()
                .unwrap(),
            nvext: None,
            metadata: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Repetition penalty must be between 0 and 2, got -3")
            );
        }

        // Logprobs: Should be a positive integer between 0 and 5
        let request = NvCreateCompletionRequest {
            inner: CreateCompletionRequest {
                model: "test-model".to_string(),
                prompt: "Hello".into(),
                logprobs: Some(6),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            metadata: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Logprobs must be between 0 and 5, got 6")
            );
        }
    }

    #[test]
    fn test_metadata_field_nested() {
        use serde_json::json;

        // Test metadata field with nested object
        let request = NvCreateCompletionRequest {
            inner: CreateCompletionRequest {
                model: "test-model".to_string(),
                prompt: "Hello".into(),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            metadata: json!({
                "user": {"id": 1, "name": "user-1"},
                "session": {"id": "session-1", "timestamp": 1640995200}
            })
            .into(),
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };

        let result = validate_completion_fields_generic(&request);
        assert!(result.is_ok());

        // Verify metadata is accessible
        assert!(request.metadata.is_some());
        assert_eq!(request.metadata.as_ref().unwrap()["user"]["id"], 1);
    }

    #[test]
    fn test_bad_base_request_for_chatcompletion() {
        // Frequency Penalty: Should be a float between -2.0 and 2.0
        let request = NvCreateChatCompletionRequest {
            inner: CreateChatCompletionRequest {
                model: "test-model".to_string(),
                messages: vec![ChatCompletionRequestMessage::User(
                    ChatCompletionRequestUserMessage {
                        content: ChatCompletionRequestUserMessageContent::Text("Hello".to_string()),
                        name: None,
                    },
                )],
                frequency_penalty: Some(-3.0),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            chat_template_args: None,
            thinking: None,
            media_io_kwargs: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };

        let result = validate_chat_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Frequency penalty must be between -2 and 2, got -3")
            );
        }

        // Presence Penalty: Should be a float between -2.0 and 2.0
        let request = NvCreateChatCompletionRequest {
            inner: CreateChatCompletionRequest {
                model: "test-model".to_string(),
                messages: vec![ChatCompletionRequestMessage::User(
                    ChatCompletionRequestUserMessage {
                        content: ChatCompletionRequestUserMessageContent::Text("Hello".to_string()),
                        name: None,
                    },
                )],
                presence_penalty: Some(-3.0),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            chat_template_args: None,
            thinking: None,
            media_io_kwargs: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_chat_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Presence penalty must be between -2 and 2, got -3")
            );
        }

        // Temperature: Should be a float between 0.0 and 2.0
        let request = NvCreateChatCompletionRequest {
            inner: CreateChatCompletionRequest {
                model: "test-model".to_string(),
                messages: vec![ChatCompletionRequestMessage::User(
                    ChatCompletionRequestUserMessage {
                        content: ChatCompletionRequestUserMessageContent::Text("Hello".to_string()),
                        name: None,
                    },
                )],
                temperature: Some(-3.0),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            chat_template_args: None,
            thinking: None,
            media_io_kwargs: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_chat_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Temperature must be between 0 and 2, got -3")
            );
        }

        // Top P: Should be a float between 0.0 and 1.0
        let request = NvCreateChatCompletionRequest {
            inner: CreateChatCompletionRequest {
                model: "test-model".to_string(),
                messages: vec![ChatCompletionRequestMessage::User(
                    ChatCompletionRequestUserMessage {
                        content: ChatCompletionRequestUserMessageContent::Text("Hello".to_string()),
                        name: None,
                    },
                )],
                top_p: Some(-3.0),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            chat_template_args: None,
            thinking: None,
            media_io_kwargs: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_chat_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Top_p must be between 0 and 1, got -3")
            );
        }

        // Repetition Penalty: Should be a float between 0.0 and 2.0
        let request = NvCreateChatCompletionRequest {
            inner: CreateChatCompletionRequest {
                model: "test-model".to_string(),
                messages: vec![ChatCompletionRequestMessage::User(
                    ChatCompletionRequestUserMessage {
                        content: ChatCompletionRequestUserMessageContent::Text("Hello".to_string()),
                        name: None,
                    },
                )],
                ..Default::default()
            },
            common: CommonExt::builder()
                .repetition_penalty(-3.0)
                .build()
                .unwrap(),
            nvext: None,
            chat_template_args: None,
            thinking: None,
            media_io_kwargs: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_chat_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Repetition penalty must be between 0 and 2, got -3")
            );
        }

        // Top Logprobs: Should be a positive integer between 0 and 20
        let request = NvCreateChatCompletionRequest {
            inner: CreateChatCompletionRequest {
                model: "test-model".to_string(),
                messages: vec![ChatCompletionRequestMessage::User(
                    ChatCompletionRequestUserMessage {
                        content: ChatCompletionRequestUserMessageContent::Text("Hello".to_string()),
                        name: None,
                    },
                )],
                top_logprobs: Some(25),
                ..Default::default()
            },
            common: Default::default(),
            nvext: None,
            chat_template_args: None,
            thinking: None,
            media_io_kwargs: None,
            return_tokens_as_token_ids: None,
            unsupported_fields: Default::default(),
        };
        let result = validate_chat_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(
                error_response.1.message,
                format!("{VALIDATION_PREFIX}Top_logprobs must be between 0 and 20, got 25")
            );
        }
    }

    #[test]
    fn test_chat_completions_unknown_fields_rejected() {
        // Test that known unsupported fields are rejected and all shown in error message
        let json = r#"{
            "messages": [{"role": "user", "content": "Hello"}],
            "model": "test-model",
            "add_special_tokens": true,
            "documents": ["doc1"],
            "chat_template": "custom"
        }"#;

        let request: NvCreateChatCompletionRequest = serde_json::from_str(json).unwrap();

        // Verify all unsupported fields were captured
        assert!(
            request
                .unsupported_fields
                .contains_key("add_special_tokens")
        );
        assert!(request.unsupported_fields.contains_key("documents"));
        assert!(request.unsupported_fields.contains_key("chat_template"));

        let result = validate_chat_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            let msg = &error_response.1.message;
            assert!(msg.contains("Unsupported parameter"));
            // Verify all fields appear in the error message
            assert!(msg.contains("add_special_tokens"));
            assert!(msg.contains("documents"));
            assert!(msg.contains("chat_template"));
        }
    }

    #[test]
    fn test_completions_unsupported_fields_rejected() {
        // Test that known unsupported fields are rejected and all shown in error message
        let json = r#"{
            "model": "test-model",
            "prompt": "Hello",
            "add_special_tokens": true,
            "response_format": {"type": "json_object"}
        }"#;

        let request: NvCreateCompletionRequest = serde_json::from_str(json).unwrap();

        // Verify both unsupported fields were captured
        assert!(
            request
                .unsupported_fields
                .contains_key("add_special_tokens")
        );
        assert!(request.unsupported_fields.contains_key("response_format"));

        let result = validate_completion_fields_generic(&request);
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            let msg = &error_response.1.message;
            assert!(msg.contains("Unsupported parameter"));
            // Verify both fields appear in error message
            assert!(msg.contains("add_special_tokens"));
            assert!(msg.contains("response_format"));
        }
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_error_event() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        // Create an error event
        let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: Some(vec!["Backend service unavailable".to_string()]),
            error: None,
        };

        let test_stream = stream::iter(vec![error_event]);
        let result = check_for_backend_error(test_stream, None).await;

        // Should return an error
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::INTERNAL_SERVER_ERROR);
            // Backend-supplied 5xx text must not be forwarded to the client.
            assert_eq!(error_response.1.message, "Internal server error");
            assert!(
                !error_response
                    .1
                    .message
                    .contains("Backend service unavailable")
            );
        }
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_typed_invalid_argument() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType};
        use futures::stream;

        for error_type in [
            ErrorType::InvalidArgument,
            ErrorType::Backend(BackendError::InvalidArgument),
        ] {
            let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
                data: None,
                id: None,
                event: Some("error".to_string()),
                comment: None,
                error: Some(
                    DynamoError::builder()
                        .error_type(error_type)
                        .message("unsupported JSON schema keyword")
                        .build(),
                ),
            };

            let result = check_for_backend_error(stream::iter(vec![error_event]), None).await;

            let error_response = match result {
                Err(error_response) => error_response,
                Ok(_) => panic!("typed invalid argument must fail"),
            };
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(error_response.1.code, StatusCode::BAD_REQUEST.as_u16());
            assert_eq!(error_response.1.error_type, "Bad Request");
            assert_eq!(error_response.1.message, "unsupported JSON schema keyword");
        }
    }

    #[tokio::test]
    async fn test_completion_backend_invalid_argument_surfaces_as_400() {
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType};
        use futures::stream;

        let error_event = Annotated::<NvCreateCompletionResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: None,
            error: Some(
                DynamoError::builder()
                    .error_type(ErrorType::Backend(BackendError::InvalidArgument))
                    .message("Dynamo's SGLang backend does not currently support logprobs >= 1")
                    .build(),
            ),
        };

        let error_response =
            match check_for_backend_error(stream::iter(vec![error_event]), None).await {
                Ok(_) => panic!("typed completion error must fail"),
                Err(error_response) => error_response,
            };

        assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
        assert_eq!(error_response.1.code, StatusCode::BAD_REQUEST.as_u16());
        assert_eq!(error_response.1.error_type, "Bad Request");
        assert!(
            error_response
                .1
                .message
                .contains("does not currently support logprobs >= 1")
        );
    }

    #[tokio::test]
    async fn test_batch_completion_checks_every_stream_for_backend_errors() {
        use dynamo_runtime::error::{BackendError, DynamoError, ErrorType};
        use futures::stream;

        let normal_event = Annotated::<NvCreateCompletionResponse> {
            data: Some(make_completion_chunk("ok", None, None)),
            id: None,
            event: None,
            comment: None,
            error: None,
        };
        let error_event = Annotated::<NvCreateCompletionResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: None,
            error: Some(
                DynamoError::builder()
                    .error_type(ErrorType::Backend(BackendError::InvalidArgument))
                    .message("invalid second prompt")
                    .build(),
            ),
        };

        let result = check_completion_batch_streams(vec![
            stream::iter(vec![normal_event]),
            stream::iter(vec![error_event]),
        ])
        .await;

        let error_response = match result {
            Ok(_) => panic!("an error in any batch prompt must fail the request"),
            Err(error_response) => error_response,
        };
        assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
        assert_eq!(error_response.1.code, StatusCode::BAD_REQUEST.as_u16());
        assert_eq!(error_response.1.error_type, "Bad Request");
        assert_eq!(error_response.1.message, "invalid second prompt");
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_json_error_and_code() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        // Create an error event with JSON payload containing error code in comment
        let error_json =
            r#"{"message":"prompt > max_seq_len","type":"Internal Server Error","code":500}"#;
        let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: Some(vec![error_json.to_string()]),
            error: None,
        };

        let test_stream = stream::iter(vec![error_event]);
        let result = check_for_backend_error(test_stream, None).await;

        // Should return an error with correct status code extracted from JSON
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::INTERNAL_SERVER_ERROR);
            // 500 backend JSON messages are sanitized to a static client
            // message; the raw payload is only logged server-side.
            assert_eq!(error_response.1.message, "Internal server error");
            assert_eq!(error_response.1.code, 500);
            assert!(!error_response.1.message.contains("prompt > max_seq_len"));
        }
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_non_client_error_code() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        // A backend asserting a non-4xx code (here 399) must not be able to
        // smuggle a sensitive message through with a non-error status:
        // anything outside the 4xx range is sanitized to 500.
        let error_json =
            r#"{"message":"panic at /srv/model.py:42","type":"Backend Error","code":399}"#;
        let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: Some(vec![error_json.to_string()]),
            error: None,
        };

        let test_stream = stream::iter(vec![error_event]);
        let result = check_for_backend_error(test_stream, None).await;

        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::INTERNAL_SERVER_ERROR);
            assert_eq!(error_response.1.code, 500);
            assert_eq!(error_response.1.message, "Internal server error");
            assert!(!error_response.1.message.contains("/srv/model.py"));
            assert!(!error_response.1.message.contains("panic"));
        }
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_503_preserves_status() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        // Backend 5xx status codes must round-trip so clients can distinguish
        // retryable overload (503) from generic 500; only the body is sanitized.
        let error_json = r#"{"message":"engine pool exhausted at /srv/engine.py:88","code":503}"#;
        let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: Some(vec![error_json.to_string()]),
            error: None,
        };

        let test_stream = stream::iter(vec![error_event]);
        let result = check_for_backend_error(test_stream, None).await;

        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::SERVICE_UNAVAILABLE);
            assert_eq!(error_response.1.code, 503);
            assert_eq!(error_response.1.message, "Internal server error");
            assert!(!error_response.1.message.contains("engine pool"));
            assert!(!error_response.1.message.contains("/srv/engine.py"));
        }
    }

    /// The streaming path must triage a backend status exactly as the unary
    /// path does. Both once called `SanitizedError::for_backend_status`, which
    /// preserves every 5xx, and only the unary path moved to
    /// `BackendStatusAction`. A backend 501 therefore answered 501 mid-stream
    /// and 500 as an `HttpError`, so the status a client saw depended on which
    /// door the same failure came through.
    ///
    /// The status codes here mirror `test_from_http_error_*` above, which is the
    /// point: the two lists must not drift apart again.
    #[tokio::test]
    async fn test_check_for_backend_error_matches_unary_triage() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        for (code, expected) in [
            (399u16, 500u16),
            (500, 500),
            (501, 500),
            (507, 500),
            (503, 503),
        ] {
            let error_json =
                format!(r#"{{"message":"engine failed at /srv/engine.py:88","code":{code}}}"#);
            let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
                data: None,
                id: None,
                event: Some("error".to_string()),
                comment: Some(vec![error_json]),
                error: None,
            };

            let result = check_for_backend_error(stream::iter(vec![error_event]), None).await;
            let Err(response) = result else {
                panic!("backend status {code} should produce an error response");
            };
            assert_eq!(response.0.as_u16(), expected, "status for backend {code}");
            assert_eq!(response.1.code, expected, "body code for backend {code}");
            // Sanitisation must survive the retriage: a coerced 5xx still hides
            // the backend's own message, which can carry filesystem paths.
            assert_eq!(response.1.message, "Internal server error");
            assert!(!response.1.message.contains("/srv/engine.py"));
        }
    }

    /// The configured overload status keeps its meaning on the streaming path,
    /// and reports as an overload rather than by its registered reason.
    #[tokio::test]
    async fn test_check_for_backend_error_preserves_overload_status() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        let overload = overload_status_code();
        let error_json = format!(
            r#"{{"message":"shedding load at /srv/pool.py:12","code":{}}}"#,
            overload.as_u16()
        );
        let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: Some(vec![error_json]),
            error: None,
        };

        let result = check_for_backend_error(stream::iter(vec![error_event]), None).await;
        let Err(response) = result else {
            panic!("an overload status should produce an error response");
        };
        assert_eq!(response.0, overload);
        assert_eq!(response.1.code, overload.as_u16());
        assert_eq!(response.1.error_type, "Overloaded");
        assert_eq!(
            classify_error_for_metrics(overload, &response.1.message),
            ErrorType::Overload
        );
        assert!(!response.1.message.contains("/srv/pool.py"));
    }

    /// `map_error_code_to_error_type` and `classify_error_for_metrics` must read
    /// the configured overload code rather than the literal 529. Both once
    /// special-cased 529 only, and both consulted `canonical_reason()` first, so
    /// a registered status such as 507 never reached the overload arm: the
    /// response said "Insufficient Storage" and the metric said `Internal`. 529
    /// hid that, because IANA does not register it.
    ///
    /// This asserts the wiring, not a non-default value. `overload_status_code`
    /// caches in a `LazyLock`, so a test cannot change it after first use, and
    /// only a process started with `DYN_HTTP_OVERLOAD_STATUS_CODE` set exercises
    /// the non-default path.
    #[test]
    fn test_overload_classification_follows_configured_code() {
        let overload = overload_status_code();
        assert_eq!(map_error_code_to_error_type(overload), "Overloaded");
        assert_eq!(
            classify_error_for_metrics(overload, "Internal server error"),
            ErrorType::Overload
        );
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_499_sanitizes_cancellation() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        // 499 falls inside is_client_error(); ensure cancellation text from
        // the backend (e.g. context IDs) cannot reach the client.
        let error_json =
            r#"{"message":"Context id abc-123 cancelled at /srv/queue.py:42","code":499}"#;
        let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: Some(vec![error_json.to_string()]),
            error: None,
        };

        let test_stream = stream::iter(vec![error_event]);
        let result = check_for_backend_error(test_stream, None).await;

        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0.as_u16(), 499);
            assert_eq!(error_response.1.code, 499);
            assert_eq!(error_response.1.message, "Request cancelled");
            assert!(!error_response.1.message.contains("abc-123"));
            assert!(!error_response.1.message.contains("/srv/queue.py"));
        }
    }

    #[tokio::test]
    async fn test_check_for_backend_error_skips_leading_annotation_frames() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        // Streams prepend a request_id annotation before forwarding engine
        // events. An immediate backend error in the next slot must still be
        // caught so a 4xx surfaces as a 4xx instead of falling through to
        // the generic fold/parse 500.
        let annotation = Annotated::<NvCreateChatCompletionStreamResponse>::from_annotation(
            ANNOTATION_REQUEST_ID,
            &"req-123".to_string(),
        )
        .expect("annotation construction should succeed");
        let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: None,
            id: None,
            event: Some("error".to_string()),
            comment: Some(vec![
                r#"{"message":"bad input from client","code":400}"#.to_string(),
            ]),
            error: None,
        };

        let test_stream = stream::iter(vec![annotation, error_event]);
        let result = check_for_backend_error(test_stream, None).await;

        assert!(
            result.is_err(),
            "annotation followed by an error event must still be detected as an error"
        );
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::BAD_REQUEST);
            assert_eq!(error_response.1.code, 400);
            assert_eq!(error_response.1.message, "bad input from client");
        }
    }

    #[tokio::test]
    async fn test_check_for_backend_error_replays_leading_annotation_frames() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use dynamo_protocols::types::CreateChatCompletionStreamResponse;
        use futures::stream::{self, StreamExt};

        // A leading annotation followed by a normal data event must yield
        // a stream that replays both, in their original order.
        let annotation = Annotated::<NvCreateChatCompletionStreamResponse>::from_annotation(
            ANNOTATION_REQUEST_ID,
            &"req-123".to_string(),
        )
        .expect("annotation construction should succeed");
        let normal_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: Some(NvCreateChatCompletionStreamResponse {
                inner: CreateChatCompletionStreamResponse {
                    id: "test-id".to_string(),
                    choices: vec![],
                    created: 0,
                    model: "test-model".to_string(),
                    system_fingerprint: None,
                    object: "chat.completion.chunk".to_string(),
                    service_tier: None,
                    usage: None,
                },
                nvext: None,
                llm_metrics: None,
            }),
            id: Some("msg-1".to_string()),
            event: None,
            comment: None,
            error: None,
        };

        let test_stream = stream::iter(vec![annotation, normal_event]);
        let result = check_for_backend_error(test_stream, None).await;

        assert!(result.is_ok());
        let mut returned: Vec<_> = result.unwrap().collect().await;
        assert_eq!(returned.len(), 2, "annotation + data event must replay");
        let first = returned.remove(0);
        assert_eq!(first.event.as_deref(), Some(ANNOTATION_REQUEST_ID));
        let second = returned.remove(0);
        assert_eq!(second.id, Some("msg-1".to_string()));
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_normal_event() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use dynamo_protocols::types::CreateChatCompletionStreamResponse;
        use futures::stream::{self, StreamExt};

        // Create a normal data event
        let normal_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: Some(NvCreateChatCompletionStreamResponse {
                inner: CreateChatCompletionStreamResponse {
                    id: "test-id".to_string(),
                    choices: vec![],
                    created: 0,
                    model: "test-model".to_string(),
                    system_fingerprint: None,
                    object: "chat.completion.chunk".to_string(),
                    service_tier: None,
                    usage: None,
                },
                nvext: None,
                llm_metrics: None,
            }),
            id: Some("msg-1".to_string()),
            event: None,
            comment: None,
            error: None,
        };

        let test_stream = stream::iter(vec![normal_event.clone()]);
        let result = check_for_backend_error(test_stream, None).await;

        // Should return Ok with the stream
        assert!(result.is_ok());
        let mut returned_stream = result.unwrap();

        // Verify we can read the event back from the stream
        let first = returned_stream.next().await;
        assert!(first.is_some());
        let first_event = first.unwrap();
        assert_eq!(first_event.id, Some("msg-1".to_string()));
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_empty_stream() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream::{self, StreamExt};

        // Create an empty stream
        let test_stream =
            stream::iter::<Vec<Annotated<NvCreateChatCompletionStreamResponse>>>(vec![]);
        let result = check_for_backend_error(test_stream, None).await;

        // Should return Ok with an empty stream
        assert!(result.is_ok());
        let mut returned_stream = result.unwrap();

        // Verify stream is empty
        let first = returned_stream.next().await;
        assert!(first.is_none());
    }

    #[tokio::test]
    async fn test_check_for_backend_error_with_comment_but_no_event_type() {
        use crate::types::openai::chat_completions::NvCreateChatCompletionStreamResponse;
        use futures::stream;

        // Create an event with comment but no event type and no data (error indicator)
        let error_event = Annotated::<NvCreateChatCompletionStreamResponse> {
            data: None,
            id: None,
            event: None,
            comment: Some(vec!["Connection timeout".to_string()]),
            error: None,
        };

        let test_stream = stream::iter(vec![error_event]);
        let result = check_for_backend_error(test_stream, None).await;

        // Should return an error based on is_backend_error_event logic
        assert!(result.is_err());
        if let Err(error_response) = result {
            assert_eq!(error_response.0, StatusCode::INTERNAL_SERVER_ERROR);
            // Backend comment text falls under the 5xx default — must be
            // sanitized so it cannot leak internals to the client.
            assert_eq!(error_response.1.message, "Internal server error");
            assert!(!error_response.1.message.contains("Connection timeout"));
        }
    }

    #[test]
    fn test_classify_error_for_metrics_validation() {
        // 400 with "Validation:" prefix to validation
        let error_type =
            classify_error_for_metrics(StatusCode::BAD_REQUEST, "Validation: Invalid parameter");
        assert_eq!(error_type, ErrorType::Validation);

        // 400 WITHOUT "Validation:" to internal (fallback)
        let error_type = classify_error_for_metrics(StatusCode::BAD_REQUEST, "Some other error");
        assert_eq!(error_type, ErrorType::Internal);
    }

    #[test]
    fn test_classify_error_for_metrics_status_codes() {
        assert_eq!(
            classify_error_for_metrics(StatusCode::NOT_FOUND, "Model not found"),
            ErrorType::NotFound
        );
        assert_eq!(
            classify_error_for_metrics(StatusCode::NOT_IMPLEMENTED, "Feature not supported"),
            ErrorType::NotImplemented
        );
        assert_eq!(
            classify_error_for_metrics(StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded"),
            ErrorType::Overload
        );
        assert_eq!(
            classify_error_for_metrics(StatusCode::SERVICE_UNAVAILABLE, "Unavailable"),
            ErrorType::Unavailable
        );
        assert_eq!(
            classify_error_for_metrics(overload_status_code(), "Overloaded"),
            ErrorType::Overload
        );
        assert_eq!(
            classify_error_for_metrics(StatusCode::INTERNAL_SERVER_ERROR, "Panic"),
            ErrorType::Internal
        );
    }

    #[test]
    fn test_classify_error_for_metrics_client_errors() {
        // Other 4xx errors should be classified as validation
        assert_eq!(
            classify_error_for_metrics(StatusCode::UNAUTHORIZED, "Unauthorized"),
            ErrorType::Validation
        );
        assert_eq!(
            classify_error_for_metrics(StatusCode::FORBIDDEN, "Forbidden"),
            ErrorType::Validation
        );
    }

    #[test]
    fn test_extract_error_type_from_response_validation() {
        let response = ErrorMessage::from_http_error(HttpError {
            code: 400,
            message: "Validation: bad input".to_string(),
        });
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::Validation
        );
    }

    #[test]
    fn test_extract_error_type_from_response_not_found() {
        let response = ErrorMessage::model_not_found();
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::NotFound
        );
    }

    #[test]
    fn test_extract_error_type_from_response_unavailable() {
        let response =
            ErrorMessage::from_model_error(&ModelManagerError::ModelUnavailable("x".to_string()));
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::Unavailable
        );
    }

    #[test]
    fn test_from_model_error_maps_correctly() {
        let not_found = ModelManagerError::ModelNotFound("x".to_string());
        assert_eq!(
            ErrorMessage::from_model_error(&not_found).0,
            StatusCode::NOT_FOUND
        );

        let unavailable = ModelManagerError::ModelUnavailable("x".to_string());
        assert_eq!(
            ErrorMessage::from_model_error(&unavailable).0,
            StatusCode::SERVICE_UNAVAILABLE
        );
    }

    /// The not-ready 503 must be customer-facing: clear and actionable, but free
    /// of internal worker-role / topology taxonomy. Whichever role is missing
    /// (prefill or decode), the client sees the same text — so the message must
    /// never name a specific role, namespace, or "worker set".
    #[test]
    fn test_model_not_ready_message_hides_internals() {
        let msg = model_not_ready_message("my-model").to_lowercase();
        for leak in [
            "prefill",
            "decode",
            "encode",
            "worker",
            "namespace",
            "needs",
        ] {
            assert!(
                !msg.contains(leak),
                "not-ready message leaks internal term `{leak}`: {msg}"
            );
        }
        // Still names the model and signals retryability.
        assert!(model_not_ready_message("my-model").contains("my-model"));
        assert!(msg.contains("retry"));
    }

    /// The dispatch-time backstop (`from_model_error` on `ModelUnavailable`) and
    /// the up-front readiness gate must speak with one voice: identical 503 body
    /// for the same "registered but not servable" condition, regardless of which
    /// role (prefill vs decode) is the missing one.
    #[test]
    fn test_unavailable_paths_share_one_message() {
        let backstop = ErrorMessage::from_model_error(&ModelManagerError::ModelUnavailable(
            "my-model".to_string(),
        ));
        assert_eq!(backstop.0, StatusCode::SERVICE_UNAVAILABLE);
        assert_eq!(backstop.1.message, model_not_ready_message("my-model"));

        // The gate constructs its body from the same canonical helper, so the
        // two paths cannot drift apart.
        let gate = ErrorMessage::service_unavailable_with_body(model_not_ready_message("my-model"));
        assert_eq!(gate.1.message, backstop.1.message);
    }

    #[test]
    fn test_extract_error_type_from_response_internal() {
        let response = ErrorMessage::internal_server_error("Something went wrong");
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::Internal
        );
    }

    /// `internal_server_error` and `internal_server_error_with_details` set
    /// `error_type`/`metric_error_type` directly, the same as
    /// `_service_unavailable` does for 503. If they instead derived those
    /// from `map_error_code_to_error_type(StatusCode::INTERNAL_SERVER_ERROR)`,
    /// an operator who set `DYN_HTTP_OVERLOAD_STATUS_CODE=500` would see every
    /// genuine internal error reported and counted as "Overloaded", though it
    /// has nothing to do with load shedding.
    #[test]
    fn test_internal_server_error_ignores_configured_overload() {
        let plain = ErrorMessage::internal_server_error("boom");
        assert_eq!(plain.1.error_type, "Internal Server Error");
        assert_eq!(plain.1.metric_error_type, Some(ErrorType::Internal));

        let with_details = ErrorMessage::internal_server_error_with_details("boom", "cause");
        assert_eq!(with_details.1.error_type, "Internal Server Error");
        assert_eq!(with_details.1.metric_error_type, Some(ErrorType::Internal));

        let sanitized = ErrorMessage::sanitized_with_details(SanitizedError::Internal, "cause");
        assert_eq!(sanitized.1.error_type, "Internal Server Error");
        assert_eq!(sanitized.1.metric_error_type, Some(ErrorType::Internal));
    }

    #[test]
    fn test_extract_error_type_from_response_not_implemented() {
        let response = ErrorMessage::not_implemented_error("Feature not available");
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::NotImplemented
        );
    }

    #[test]
    fn unsupported_content_responses_conversion_errors_are_not_implemented() {
        let response = responses_conversion_error_response(
            ResponsesConversionError::UnsupportedContent("feature not available".to_string())
                .into(),
        );

        assert_eq!(response.0, StatusCode::BAD_REQUEST);
        assert_eq!(response.1.error_type, "Bad Request");
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::NotImplemented
        );
    }

    #[test]
    fn untyped_responses_conversion_errors_remain_internal() {
        let response = responses_conversion_error_response(anyhow::anyhow!(
            "internal response conversion details"
        ));

        assert_eq!(response.0, StatusCode::INTERNAL_SERVER_ERROR);
        assert_eq!(response.1.message, "Failed to convert responses request");
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::Internal
        );
    }

    #[test]
    fn invalid_responses_conversion_errors_are_client_errors() {
        let response = responses_conversion_error_response(
            ResponsesConversionError::InvalidArgument("ambiguous tools".to_string()).into(),
        );

        assert_eq!(response.0, StatusCode::BAD_REQUEST);
        assert_eq!(
            extract_error_type_from_response(&response),
            ErrorType::Validation
        );
    }

    // ── streaming dispatch tests ──────────────────────────────────────

    use std::collections::{HashMap, HashSet};

    use dynamo_protocols::types::{
        ChatChoiceStream, ChatCompletionMessageToolCallChunk, ChatCompletionStreamResponseDelta,
        ChatCompletionStreamResponseDeltaFunctionCall, CreateChatCompletionStreamResponse,
        FinishReason, FunctionCallStream, FunctionType, Role,
    };
    use dynamo_runtime::protocols::annotated::Annotated;

    /// Extract the JSON data payload from an SSE Event's Debug output.
    ///
    /// `axum::response::sse::Event` doesn't expose its fields publicly and doesn't
    /// implement `Display` (the wire format is only produced during response
    /// serialization). The `Debug` representation includes the event name and data
    /// string, so we parse it here.
    ///
    /// WARNING: Coupled to axum's internal Debug format for `Event`. If an axum
    /// upgrade changes the Debug output, these tests will break. Preferred over
    /// spinning up an actual SSE stream for unit test simplicity.
    fn extract_sse_data_json(event: &axum::response::sse::Event) -> serde_json::Value {
        // The Event Debug format is:
        //   Event { buffer: b"event: <name>\ndata: <json>\n", flags: ... }
        // We extract the JSON after "data: " and unescape the byte-string encoding.
        let debug = format!("{:?}", event);

        let data_marker = "data: ";
        let after_data = debug
            .find(data_marker)
            .map(|p| p + data_marker.len())
            .expect("no 'data: ' in Event debug output");

        let rest = &debug[after_data..];
        let json_start = rest.find('{').expect("no JSON object after data:");

        let mut depth = 0i32;
        let mut json_end = 0;
        for (i, b) in rest[json_start..].bytes().enumerate() {
            match b {
                b'{' => depth += 1,
                b'}' => {
                    depth -= 1;
                    if depth == 0 {
                        json_end = json_start + i + 1;
                        break;
                    }
                }
                _ => {}
            }
        }

        let raw = &rest[json_start..json_end];

        // Unescape byte-string Debug format:
        // \\\\\" -> PLACEHOLDER (nested escaped quotes in JSON string values)
        // \\\"   -> "           (structural quotes)
        // Then restore: PLACEHOLDER -> \"
        let s = raw
            .replace("\\\\\\\"", "\x00NESTED\x00")
            .replace("\\\"", "\"")
            .replace("\x00NESTED\x00", "\\\"");

        // Handle \\xHH byte sequences (non-ASCII in Debug byte-string format)
        let mut result = Vec::new();
        let sbytes = s.as_bytes();
        let mut idx = 0;
        while idx < sbytes.len() {
            if idx + 3 < sbytes.len()
                && sbytes[idx] == b'\\'
                && sbytes[idx + 1] == b'x'
                && let Ok(v) = u8::from_str_radix(
                    std::str::from_utf8(&sbytes[idx + 2..idx + 4]).unwrap_or(""),
                    16,
                )
            {
                result.push(v);
                idx += 4;
                continue;
            }
            result.push(sbytes[idx]);
            idx += 1;
        }

        let final_str = String::from_utf8_lossy(&result);
        serde_json::from_str(&final_str).unwrap_or_else(|e| {
            panic!(
                "failed to parse JSON from Event: {e}\nraw: {raw}\nunescaped: {s}\nfinal: {final_str}"
            )
        })
    }

    /// Assert that an SSE Event has the expected event type name.
    /// Uses "event: <name>\n" pattern to avoid substring false-matches.
    fn assert_event_type(event: &axum::response::sse::Event, expected: &str) {
        let debug = format!("{:?}", event);
        let pattern = format!("event: {expected}\\n");
        assert!(
            debug.contains(&pattern),
            "expected event type '{expected}' not found in: {debug}"
        );
    }

    /// Build a minimal Annotated<Response> with the given choices.
    fn make_stream_response(
        choices: Vec<ChatChoiceStream>,
    ) -> Annotated<NvCreateChatCompletionStreamResponse> {
        let response = NvCreateChatCompletionStreamResponse {
            inner: CreateChatCompletionStreamResponse {
                id: "test-id".to_string(),
                choices,
                created: 0,
                model: "test-model".to_string(),
                system_fingerprint: None,
                object: "chat.completion.chunk".to_string(),
                usage: None,
                service_tier: None,
            },
            nvext: None,
            llm_metrics: None,
        };
        Annotated {
            id: Some("test-id".to_string()),
            data: Some(response),
            event: None,
            comment: None,
            error: None,
        }
    }

    fn collect_tool_dispatch_events(
        response: &Annotated<NvCreateChatCompletionStreamResponse>,
        dispatched_ids: &mut HashSet<(u32, String)>,
    ) -> Vec<Result<Event, axum::Error>> {
        let mut events = Vec::new();
        streaming_tool_dispatch_events(response, dispatched_ids, &mut events);
        events
    }

    fn collect_reasoning_dispatch_events(
        response: &Annotated<NvCreateChatCompletionStreamResponse>,
        buffers: &mut HashMap<u32, String>,
    ) -> Vec<Result<Event, axum::Error>> {
        let mut events = Vec::new();
        accumulate_reasoning_dispatch(response, buffers, &mut events);
        events
    }

    fn make_choice_with_reasoning(
        index: u32,
        reasoning: Option<&str>,
        finish: Option<FinishReason>,
    ) -> ChatChoiceStream {
        #[allow(deprecated)]
        ChatChoiceStream {
            index,
            delta: ChatCompletionStreamResponseDelta {
                content: None,
                function_call: None,
                tool_calls: None,
                role: None,
                refusal: None,
                reasoning_content: reasoning.map(|s| s.to_string()),
            },
            finish_reason: finish,
            logprobs: None,
        }
    }

    fn make_choice_with_tool_call(
        index: u32,
        id: Option<&str>,
        name: Option<&str>,
        arguments: Option<&str>,
    ) -> ChatChoiceStream {
        let tool_call = ChatCompletionMessageToolCallChunk {
            index: 0,
            id: id.map(|s| s.to_string()),
            r#type: Some(FunctionType::Function),
            function: Some(FunctionCallStream {
                name: name.map(|s| s.to_string()),
                arguments: arguments.map(|s| s.to_string()),
            }),
        };
        #[allow(deprecated)]
        ChatChoiceStream {
            index,
            delta: ChatCompletionStreamResponseDelta {
                content: None,
                function_call: None,
                tool_calls: Some(vec![tool_call]),
                role: None,
                refusal: None,
                reasoning_content: None,
            },
            finish_reason: None,
            logprobs: None,
        }
    }

    // ── streaming_tool_dispatch_events tests ──

    #[test]
    fn test_tool_dispatch_emits_event_for_complete_tool_call() {
        let response = make_stream_response(vec![make_choice_with_tool_call(
            0,
            Some("call_123"),
            Some("get_weather"),
            Some(r#"{"city":"Paris"}"#),
        )]);

        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert_eq!(events.len(), 1);

        let event = events[0].as_ref().unwrap();
        assert_event_type(event, "tool_call_dispatch");
        let json = extract_sse_data_json(event);
        assert_eq!(json["choice_index"], 0);
        assert_eq!(json["tool_call"]["id"], "call_123");
        assert_eq!(json["tool_call"]["function"]["name"], "get_weather");
        assert_eq!(
            json["tool_call"]["function"]["arguments"],
            r#"{"city":"Paris"}"#
        );
    }

    #[test]
    fn test_tool_dispatch_skips_incomplete_tool_call_no_id() {
        let response = make_stream_response(vec![make_choice_with_tool_call(
            0,
            None, // no id
            Some("get_weather"),
            Some(r#"{"city":"Paris"}"#),
        )]);

        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert!(events.is_empty(), "should not dispatch without id");
    }

    #[test]
    fn test_tool_dispatch_skips_incomplete_tool_call_no_name() {
        let response = make_stream_response(vec![make_choice_with_tool_call(
            0,
            Some("call_123"),
            None, // no name
            Some(r#"{"city":"Paris"}"#),
        )]);

        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert!(events.is_empty(), "should not dispatch without name");
    }

    #[test]
    fn test_tool_dispatch_skips_incomplete_tool_call_no_arguments() {
        let response = make_stream_response(vec![make_choice_with_tool_call(
            0,
            Some("call_123"),
            Some("get_weather"),
            None, // no arguments
        )]);

        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert!(events.is_empty(), "should not dispatch without arguments");
    }

    #[test]
    fn test_tool_dispatch_multiple_tool_calls() {
        let tc1 = ChatCompletionMessageToolCallChunk {
            index: 0,
            id: Some("call_1".to_string()),
            r#type: Some(FunctionType::Function),
            function: Some(FunctionCallStream {
                name: Some("get_weather".to_string()),
                arguments: Some(r#"{"city":"Paris"}"#.to_string()),
            }),
        };
        let tc2 = ChatCompletionMessageToolCallChunk {
            index: 1,
            id: Some("call_2".to_string()),
            r#type: Some(FunctionType::Function),
            function: Some(FunctionCallStream {
                name: Some("get_time".to_string()),
                arguments: Some(r#"{"tz":"UTC"}"#.to_string()),
            }),
        };
        #[allow(deprecated)]
        let choice = ChatChoiceStream {
            index: 0,
            delta: ChatCompletionStreamResponseDelta {
                content: None,
                function_call: None,
                tool_calls: Some(vec![tc1, tc2]),
                role: None,
                refusal: None,
                reasoning_content: None,
            },
            finish_reason: None,
            logprobs: None,
        };

        let response = make_stream_response(vec![choice]);
        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert_eq!(events.len(), 2, "should dispatch both tool calls");

        // Verify each dispatched event has the correct tool call data
        let json0 = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json0["tool_call"]["id"], "call_1");
        assert_eq!(json0["tool_call"]["function"]["name"], "get_weather");

        let json1 = extract_sse_data_json(events[1].as_ref().unwrap());
        assert_eq!(json1["tool_call"]["id"], "call_2");
        assert_eq!(json1["tool_call"]["function"]["name"], "get_time");
    }

    #[test]
    fn test_tool_dispatch_no_data() {
        let response: Annotated<NvCreateChatCompletionStreamResponse> = Annotated {
            id: Some("test".to_string()),
            data: None,
            event: None,
            comment: None,
            error: None,
        };
        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert!(events.is_empty());
    }

    #[test]
    fn test_tool_dispatch_empty_choices() {
        let response = make_stream_response(vec![]);
        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert!(events.is_empty());
    }

    #[test]
    fn test_tool_dispatch_mixed_complete_and_incomplete() {
        // One complete tool call and one incomplete (missing arguments = streaming delta).
        // Only the complete one should dispatch.
        let complete = ChatCompletionMessageToolCallChunk {
            index: 0,
            id: Some("call_complete".to_string()),
            r#type: Some(FunctionType::Function),
            function: Some(FunctionCallStream {
                name: Some("get_weather".to_string()),
                arguments: Some(r#"{"city":"Paris"}"#.to_string()),
            }),
        };
        let incomplete = ChatCompletionMessageToolCallChunk {
            index: 1,
            id: Some("call_partial".to_string()),
            r#type: Some(FunctionType::Function),
            function: Some(FunctionCallStream {
                name: Some("search".to_string()),
                arguments: None, // still streaming
            }),
        };
        #[allow(deprecated)]
        let choice = ChatChoiceStream {
            index: 0,
            delta: ChatCompletionStreamResponseDelta {
                content: None,
                function_call: None,
                tool_calls: Some(vec![complete, incomplete]),
                role: None,
                refusal: None,
                reasoning_content: None,
            },
            finish_reason: None,
            logprobs: None,
        };

        let response = make_stream_response(vec![choice]);
        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert_eq!(
            events.len(),
            1,
            "only the complete tool call should dispatch"
        );

        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json["tool_call"]["id"], "call_complete");
    }

    #[test]
    fn test_tool_dispatch_function_none() {
        // Tool call chunk with function: None — should not dispatch and should not panic.
        let tool_call = ChatCompletionMessageToolCallChunk {
            index: 0,
            id: Some("call_999".to_string()),
            r#type: Some(FunctionType::Function),
            function: None,
        };
        #[allow(deprecated)]
        let choice = ChatChoiceStream {
            index: 0,
            delta: ChatCompletionStreamResponseDelta {
                content: None,
                function_call: None,
                tool_calls: Some(vec![tool_call]),
                role: None,
                refusal: None,
                reasoning_content: None,
            },
            finish_reason: None,
            logprobs: None,
        };

        let response = make_stream_response(vec![choice]);
        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert!(events.is_empty(), "function: None should not dispatch");
    }

    #[test]
    fn test_tool_dispatch_empty_arguments_still_dispatches() {
        // arguments: Some("") is considered complete — intentional.
        // Some backends emit empty-string arguments for parameterless tools.
        let response = make_stream_response(vec![make_choice_with_tool_call(
            0,
            Some("call_empty"),
            Some("no_params_tool"),
            Some(""),
        )]);

        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert_eq!(events.len(), 1, "empty arguments should still dispatch");

        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json["tool_call"]["id"], "call_empty");
        assert_eq!(json["tool_call"]["function"]["name"], "no_params_tool");
        assert_eq!(json["tool_call"]["function"]["arguments"], "");
    }

    #[test]
    fn test_tool_dispatch_n_greater_than_1_includes_choice_index() {
        // Regression test for #12676: with n > 1, identical tool-call ids from different
        // choices must each dispatch with their own choice_index.
        let choice_0 = make_choice_with_tool_call(
            0,
            Some("call_1"),
            Some("get_weather"),
            Some(r#"{"city":"Paris"}"#),
        );
        let choice_1 = make_choice_with_tool_call(
            1,
            Some("call_1"),
            Some("get_time"),
            Some(r#"{"tz":"UTC"}"#),
        );

        let response = make_stream_response(vec![choice_0, choice_1]);
        let events = collect_tool_dispatch_events(&response, &mut HashSet::new());
        assert_eq!(events.len(), 2, "should dispatch from both choices");

        let json0 = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json0["choice_index"], 0);
        assert_eq!(json0["tool_call"]["id"], "call_1");

        let json1 = extract_sse_data_json(events[1].as_ref().unwrap());
        assert_eq!(json1["choice_index"], 1);
        assert_eq!(json1["tool_call"]["id"], "call_1");
    }

    #[test]
    fn test_tool_dispatch_dedup_skips_already_dispatched_id() {
        // Simulate a backend that sends the same complete tool call in two consecutive chunks.
        // The HashSet should prevent the second dispatch.
        let response = make_stream_response(vec![make_choice_with_tool_call(
            0,
            Some("call_dup"),
            Some("get_weather"),
            Some(r#"{"city":"Paris"}"#),
        )]);

        let mut dispatched = HashSet::new();

        // First call — should dispatch
        let events = collect_tool_dispatch_events(&response, &mut dispatched);
        assert_eq!(events.len(), 1);

        // Second call with same response — should be deduped
        let events = collect_tool_dispatch_events(&response, &mut dispatched);
        assert!(events.is_empty(), "duplicate id should not dispatch twice");
    }

    // ── accumulate_reasoning_dispatch tests ──

    #[test]
    fn test_reasoning_dispatch_accumulates_and_emits_once() {
        let mut buffers: HashMap<u32, String> = HashMap::new();

        // Chunk 1: reasoning token "Let me"
        let r1 = make_stream_response(vec![make_choice_with_reasoning(0, Some("Let me"), None)]);
        let events = collect_reasoning_dispatch_events(&r1, &mut buffers);
        assert!(
            events.is_empty(),
            "should not emit yet — still accumulating"
        );
        assert_eq!(buffers.get(&0).map(|s| s.as_str()), Some("Let me"));

        // Chunk 2: reasoning token " think"
        let r2 = make_stream_response(vec![make_choice_with_reasoning(0, Some(" think"), None)]);
        let events = collect_reasoning_dispatch_events(&r2, &mut buffers);
        assert!(
            events.is_empty(),
            "should not emit yet — still accumulating"
        );
        assert_eq!(buffers.get(&0).map(|s| s.as_str()), Some("Let me think"));

        // Chunk 3: reasoning ends (None), meaning normal content follows
        let r3 = make_stream_response(vec![make_choice_with_reasoning(0, None, None)]);
        let events = collect_reasoning_dispatch_events(&r3, &mut buffers);
        assert_eq!(events.len(), 1, "should emit single reasoning_dispatch");

        let event = events[0].as_ref().unwrap();
        assert_event_type(event, "reasoning_dispatch");
        let json = extract_sse_data_json(event);
        assert_eq!(json["reasoning_content"], "Let me think");
        assert_eq!(json["index"], 0);

        // Buffer for choice 0 should be cleared (removed or empty)
        assert!(
            buffers.get(&0).is_none_or(|s| s.is_empty()),
            "buffer should be cleared after emit"
        );
    }

    #[test]
    fn test_reasoning_dispatch_flushes_on_finish_reason() {
        let mut buffers: HashMap<u32, String> = HashMap::new();

        // Chunk 1: reasoning token
        let r1 = make_stream_response(vec![make_choice_with_reasoning(
            0,
            Some("Thinking..."),
            None,
        )]);
        collect_reasoning_dispatch_events(&r1, &mut buffers);

        // Chunk 2: finish_reason=length while still in reasoning (max_tokens hit)
        let r2 = make_stream_response(vec![make_choice_with_reasoning(
            0,
            Some(" more"),
            Some(FinishReason::Length),
        )]);
        let events = collect_reasoning_dispatch_events(&r2, &mut buffers);
        assert_eq!(events.len(), 1, "should flush on finish_reason");

        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json["reasoning_content"], "Thinking... more");
    }

    #[test]
    fn test_reasoning_dispatch_flushes_on_stop() {
        let mut buffers: HashMap<u32, String> = HashMap::new();

        // Chunk 1: reasoning token
        let r1 = make_stream_response(vec![make_choice_with_reasoning(
            0,
            Some("Analysis complete"),
            None,
        )]);
        collect_reasoning_dispatch_events(&r1, &mut buffers);

        // Chunk 2: finish_reason=stop while still in reasoning
        let r2 = make_stream_response(vec![make_choice_with_reasoning(
            0,
            Some("."),
            Some(FinishReason::Stop),
        )]);
        let events = collect_reasoning_dispatch_events(&r2, &mut buffers);
        assert_eq!(events.len(), 1, "should flush on FinishReason::Stop");

        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json["reasoning_content"], "Analysis complete.");
    }

    #[test]
    fn test_reasoning_dispatch_no_reasoning_no_event() {
        let mut buffers: HashMap<u32, String> = HashMap::new();

        // Chunk with no reasoning content at all
        let r = make_stream_response(vec![make_choice_with_reasoning(0, None, None)]);
        let events = collect_reasoning_dispatch_events(&r, &mut buffers);
        assert!(events.is_empty(), "no reasoning content = no event");
    }

    #[test]
    fn test_reasoning_dispatch_empty_string_not_accumulated() {
        let mut buffers: HashMap<u32, String> = HashMap::new();

        // Chunk with empty string reasoning (treated as no-reasoning)
        let r = make_stream_response(vec![make_choice_with_reasoning(0, Some(""), None)]);
        let events = collect_reasoning_dispatch_events(&r, &mut buffers);
        assert!(events.is_empty());
        assert!(
            buffers.get(&0).is_none_or(|s| s.is_empty()),
            "empty string should not accumulate"
        );
    }

    #[test]
    fn test_reasoning_dispatch_no_data() {
        let mut buffers: HashMap<u32, String> = HashMap::new();
        let response: Annotated<NvCreateChatCompletionStreamResponse> = Annotated {
            id: Some("test".to_string()),
            data: None,
            event: None,
            comment: None,
            error: None,
        };
        let events = collect_reasoning_dispatch_events(&response, &mut buffers);
        assert!(events.is_empty());
    }

    #[test]
    fn test_reasoning_dispatch_empty_choices() {
        let mut buffers: HashMap<u32, String> = HashMap::new();
        let response = make_stream_response(vec![]);
        let events = collect_reasoning_dispatch_events(&response, &mut buffers);
        assert!(events.is_empty());
    }

    #[test]
    fn test_reasoning_dispatch_multi_choice_independent_buffers() {
        let mut buffers: HashMap<u32, String> = HashMap::new();

        // Both choices emit reasoning in same chunk
        let r1 = make_stream_response(vec![
            make_choice_with_reasoning(0, Some("Thinking A"), None),
            make_choice_with_reasoning(1, Some("Thinking B"), None),
        ]);
        let events = collect_reasoning_dispatch_events(&r1, &mut buffers);
        assert!(events.is_empty(), "both still accumulating");
        assert_eq!(buffers.get(&0).map(|s| s.as_str()), Some("Thinking A"));
        assert_eq!(buffers.get(&1).map(|s| s.as_str()), Some("Thinking B"));

        // Choice 0 stops reasoning, choice 1 continues
        let r2 = make_stream_response(vec![
            make_choice_with_reasoning(0, None, None),
            make_choice_with_reasoning(1, Some(" more"), None),
        ]);
        let events = collect_reasoning_dispatch_events(&r2, &mut buffers);
        assert_eq!(events.len(), 1, "only choice 0 should emit");
        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json["reasoning_content"], "Thinking A");
        assert_eq!(json["index"], 0);

        // Choice 1 stops reasoning
        let r3 = make_stream_response(vec![make_choice_with_reasoning(1, None, None)]);
        let events = collect_reasoning_dispatch_events(&r3, &mut buffers);
        assert_eq!(events.len(), 1, "choice 1 should emit");
        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json["reasoning_content"], "Thinking B more");
        assert_eq!(json["index"], 1);
    }

    #[test]
    fn test_reasoning_dispatch_multiple_blocks() {
        // Reasoning -> emit -> more reasoning -> emit again.
        // Verifies that after the buffer is cleared, a new reasoning block
        // accumulates independently.
        let mut buffers: HashMap<u32, String> = HashMap::new();

        // First reasoning block
        let r1 = make_stream_response(vec![make_choice_with_reasoning(0, Some("First"), None)]);
        collect_reasoning_dispatch_events(&r1, &mut buffers);

        let r2 = make_stream_response(vec![make_choice_with_reasoning(0, None, None)]);
        let events = collect_reasoning_dispatch_events(&r2, &mut buffers);
        assert_eq!(events.len(), 1);
        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json["reasoning_content"], "First");

        // Second reasoning block — buffer was cleared, should accumulate fresh
        let r3 = make_stream_response(vec![make_choice_with_reasoning(0, Some("Second"), None)]);
        collect_reasoning_dispatch_events(&r3, &mut buffers);

        let r4 = make_stream_response(vec![make_choice_with_reasoning(0, None, None)]);
        let events = collect_reasoning_dispatch_events(&r4, &mut buffers);
        assert_eq!(events.len(), 1);
        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(
            json["reasoning_content"], "Second",
            "second emit should only contain second block's content"
        );
    }

    #[test]
    fn test_reasoning_dispatch_unicode() {
        // Verify that CJK characters and emoji survive the JSON roundtrip.
        let mut buffers: HashMap<u32, String> = HashMap::new();

        let r1 = make_stream_response(vec![make_choice_with_reasoning(
            0,
            Some("让我想想 🤔"),
            None,
        )]);
        collect_reasoning_dispatch_events(&r1, &mut buffers);

        let r2 = make_stream_response(vec![make_choice_with_reasoning(
            0,
            Some(" 分析完成 ✅"),
            None,
        )]);
        collect_reasoning_dispatch_events(&r2, &mut buffers);

        let r3 = make_stream_response(vec![make_choice_with_reasoning(0, None, None)]);
        let events = collect_reasoning_dispatch_events(&r3, &mut buffers);
        assert_eq!(events.len(), 1);

        let json = extract_sse_data_json(events[0].as_ref().unwrap());
        assert_eq!(json["reasoning_content"], "让我想想 🤔 分析完成 ✅");
    }

    /// Build a single-choice `NvCreateChatCompletionStreamResponse`.
    #[allow(clippy::too_many_arguments)]
    fn make_delta(
        content: Option<&str>,
        reasoning: Option<&str>,
        tool_calls: Option<Vec<ChatCompletionMessageToolCallChunk>>,
        finish: Option<FinishReason>,
        usage: Option<dynamo_protocols::types::CompletionUsage>,
        role: Option<Role>,
        refusal: Option<&str>,
        function_call: Option<ChatCompletionStreamResponseDeltaFunctionCall>,
    ) -> NvCreateChatCompletionStreamResponse {
        use dynamo_protocols::types::ChatCompletionMessageContent;
        #[allow(deprecated)]
        let choice = ChatChoiceStream {
            index: 0,
            delta: ChatCompletionStreamResponseDelta {
                content: content.map(|s| ChatCompletionMessageContent::Text(s.to_string())),
                function_call,
                tool_calls,
                role,
                refusal: refusal.map(|s| s.to_string()),
                reasoning_content: reasoning.map(|s| s.to_string()),
            },
            finish_reason: finish,
            logprobs: None,
        };
        NvCreateChatCompletionStreamResponse {
            inner: CreateChatCompletionStreamResponse {
                id: "test".to_string(),
                choices: vec![choice],
                created: 0,
                model: "m".to_string(),
                system_fingerprint: None,
                object: "chat.completion.chunk".to_string(),
                usage,
                service_tier: None,
            },
            nvext: None,
            llm_metrics: None,
        }
    }

    #[test]
    fn test_is_empty_stream_response() {
        // Empty: all-None, no finish, no usage
        assert!(
            is_empty_stream_response(&make_delta(None, None, None, None, None, None, None, None)),
            "all-None delta → empty",
        );

        // Not empty: has content
        assert!(
            !is_empty_stream_response(&make_delta(
                Some("hi"),
                None,
                None,
                None,
                None,
                None,
                None,
                None
            )),
            "content present → not empty",
        );

        // Not empty: has reasoning
        assert!(
            !is_empty_stream_response(&make_delta(
                None,
                Some("thinking"),
                None,
                None,
                None,
                None,
                None,
                None
            )),
            "reasoning present → not empty",
        );

        // Not empty: has finish_reason
        assert!(
            !is_empty_stream_response(&make_delta(
                None,
                None,
                None,
                Some(FinishReason::Stop),
                None,
                None,
                None,
                None,
            )),
            "finish_reason → not empty",
        );

        // Not empty: has tool_calls
        let tc = vec![ChatCompletionMessageToolCallChunk {
            index: 0,
            id: Some("call_1".to_string()),
            r#type: Some(FunctionType::Function),
            function: Some(FunctionCallStream {
                name: Some("f".to_string()),
                arguments: Some("{}".to_string()),
            }),
        }];
        assert!(
            !is_empty_stream_response(&make_delta(
                None,
                None,
                Some(tc),
                None,
                None,
                None,
                None,
                None
            )),
            "tool_calls present → not empty",
        );

        // Not empty: usage present
        let usage = dynamo_protocols::types::CompletionUsage {
            prompt_tokens: 10,
            completion_tokens: 5,
            total_tokens: 15,
            prompt_tokens_details: None,
            completion_tokens_details: None,
        };
        assert!(
            !is_empty_stream_response(&make_delta(
                None,
                None,
                None,
                None,
                Some(usage),
                None,
                None,
                None
            )),
            "usage present → not empty",
        );

        // Role-only: not empty; duplicate roles are removed before this predicate.
        assert!(
            !is_empty_stream_response(&make_delta(
                None,
                None,
                None,
                None,
                None,
                Some(Role::Assistant),
                None,
                None,
            )),
            "role-only → not empty",
        );

        // Not empty: has refusal
        assert!(
            !is_empty_stream_response(&make_delta(
                None,
                None,
                None,
                None,
                None,
                None,
                Some("I can't help with that"),
                None,
            )),
            "refusal present → not empty",
        );

        // Not empty: has function_call (deprecated but still in the struct)
        assert!(
            !is_empty_stream_response(&make_delta(
                None,
                None,
                None,
                None,
                None,
                None,
                None,
                Some(ChatCompletionStreamResponseDeltaFunctionCall {
                    name: Some("my_fn".to_string()),
                    arguments: Some("{}".to_string()),
                }),
            )),
            "function_call present → not empty",
        );
    }

    #[test]
    fn test_deduplicate_stream_roles_preserves_only_first_role_per_choice() {
        let mut emitted_roles = HashSet::new();
        let mut first_choice_zero = make_delta(
            None,
            Some("thinking"),
            None,
            None,
            None,
            Some(Role::Assistant),
            None,
            None,
        );
        let mut first_choice_one = make_delta(
            None,
            None,
            None,
            None,
            None,
            Some(Role::Assistant),
            None,
            None,
        );
        first_choice_one.inner.choices[0].index = 1;
        let mut second_choice_zero = make_delta(
            None,
            None,
            None,
            None,
            None,
            Some(Role::Assistant),
            None,
            None,
        );
        let mut second_choice_one = make_delta(
            None,
            None,
            None,
            None,
            None,
            Some(Role::Assistant),
            None,
            None,
        );
        second_choice_one.inner.choices[0].index = 1;

        deduplicate_stream_roles(&mut first_choice_zero, &mut emitted_roles);
        deduplicate_stream_roles(&mut first_choice_one, &mut emitted_roles);
        deduplicate_stream_roles(&mut second_choice_zero, &mut emitted_roles);
        deduplicate_stream_roles(&mut second_choice_one, &mut emitted_roles);

        assert_eq!(
            first_choice_zero.inner.choices[0].delta.role,
            Some(Role::Assistant)
        );
        assert_eq!(
            first_choice_one.inner.choices[0].delta.role,
            Some(Role::Assistant)
        );
        assert_eq!(second_choice_zero.inner.choices[0].delta.role, None);
        assert_eq!(second_choice_one.inner.choices[0].delta.role, None);
    }

    #[test]
    fn test_chat_predicate_filters_text_empty_string() {
        use dynamo_protocols::types::{
            ChatChoiceLogprobs, ChatCompletionMessageContent, ChatCompletionResponseContentPart,
            ChatCompletionResponseContentPartText, ChatCompletionTokenLogprob,
        };

        // `Text("")` arises during multi-byte UTF-8 token assembly and must be
        // filtered, matching `is_empty_completion_stream_response`'s `""` case.
        let resp = make_delta(Some(""), None, None, None, None, None, None, None);
        assert!(
            is_empty_stream_response(&resp),
            "Text(\"\") delta should be filtered as empty",
        );

        // Structurally empty multimodal `Parts(vec![])` is also empty.
        let mut resp = make_delta(None, None, None, None, None, None, None, None);
        resp.inner.choices[0].delta.content = Some(ChatCompletionMessageContent::Parts(Vec::new()));
        assert!(
            is_empty_stream_response(&resp),
            "Parts(vec![]) delta should be filtered as empty",
        );

        // Non-empty multimodal Parts must be preserved.
        let mut resp = make_delta(None, None, None, None, None, None, None, None);
        resp.inner.choices[0].delta.content = Some(ChatCompletionMessageContent::Parts(vec![
            ChatCompletionResponseContentPart::Text(ChatCompletionResponseContentPartText {
                text: "hi".to_string(),
            }),
        ]));
        assert!(
            !is_empty_stream_response(&resp),
            "Parts with content must not be filtered",
        );

        // Empty content alongside a semantic field (finish_reason) must survive.
        let resp = make_delta(
            Some(""),
            None,
            None,
            Some(FinishReason::Stop),
            None,
            None,
            None,
            None,
        );
        assert!(
            !is_empty_stream_response(&resp),
            "Text(\"\") + finish_reason must not be filtered",
        );

        // Per-token logprobs can arrive while content is still `Text("")` during
        // multi-byte assembly; that payload is meaningful and must survive.
        let mut resp = make_delta(Some(""), None, None, None, None, None, None, None);
        resp.inner.choices[0].logprobs = Some(ChatChoiceLogprobs {
            content: Some(vec![ChatCompletionTokenLogprob {
                token: "h".to_string(),
                logprob: -0.5,
                token_id: None,
                bytes: Some(vec![104]),
                top_logprobs: vec![],
            }]),
            refusal: None,
        });
        assert!(
            !is_empty_stream_response(&resp),
            "Text(\"\") + logprobs must not be filtered",
        );
    }

    // ── completions empty-stream-response tests ──────────────────────

    use dynamo_protocols::types::{Choice, CompletionFinishReason, CreateCompletionResponse};

    /// Build a single-choice `NvCreateCompletionResponse`.
    fn make_completion_chunk(
        text: &str,
        finish: Option<CompletionFinishReason>,
        usage: Option<dynamo_protocols::types::CompletionUsage>,
    ) -> NvCreateCompletionResponse {
        let choice = Choice {
            text: text.to_string(),
            index: 0,
            logprobs: None,
            finish_reason: finish,
        };
        NvCreateCompletionResponse {
            inner: CreateCompletionResponse {
                id: "test".to_string(),
                choices: vec![choice],
                created: 0,
                model: "m".to_string(),
                system_fingerprint: None,
                object: "text_completion".to_string(),
                usage,
            },
            nvext: None,
        }
    }

    #[test]
    fn test_is_empty_completion_stream_response() {
        // Empty: no text, no finish, no usage
        assert!(
            is_empty_completion_stream_response(&make_completion_chunk("", None, None)),
            "empty text, no finish → empty",
        );

        // Not empty: has text
        assert!(
            !is_empty_completion_stream_response(&make_completion_chunk("hi", None, None)),
            "text present → not empty",
        );

        // Not empty: has finish_reason
        assert!(
            !is_empty_completion_stream_response(&make_completion_chunk(
                "",
                Some(CompletionFinishReason::Stop),
                None,
            )),
            "finish_reason → not empty",
        );

        // Not empty: usage present
        let usage = dynamo_protocols::types::CompletionUsage {
            prompt_tokens: 10,
            completion_tokens: 5,
            total_tokens: 15,
            prompt_tokens_details: None,
            completion_tokens_details: None,
        };
        assert!(
            !is_empty_completion_stream_response(&make_completion_chunk("", None, Some(usage))),
            "usage present → not empty",
        );
    }

    fn make_completion_usage_chunk(
        id: &str,
        usage: dynamo_protocols::types::CompletionUsage,
    ) -> NvCreateCompletionResponse {
        NvCreateCompletionResponse {
            inner: CreateCompletionResponse {
                id: id.to_string(),
                choices: vec![],
                created: 0,
                model: "m".to_string(),
                system_fingerprint: None,
                object: "text_completion".to_string(),
                usage: Some(usage),
            },
            nvext: None,
        }
    }

    #[tokio::test]
    async fn batch_completion_usage_is_aggregated_once() {
        use dynamo_protocols::types::{
            CompletionTokensDetails, CompletionUsage, PromptTokensDetails,
        };

        let continuous_usage = CompletionUsage {
            prompt_tokens: 3,
            completion_tokens: 1,
            total_tokens: 4,
            prompt_tokens_details: None,
            completion_tokens_details: None,
        };
        let first_usage = CompletionUsage {
            prompt_tokens: 3,
            completion_tokens: 2,
            total_tokens: 5,
            prompt_tokens_details: Some(PromptTokensDetails {
                audio_tokens: Some(1),
                cached_tokens: Some(2),
            }),
            completion_tokens_details: Some(CompletionTokensDetails {
                accepted_prediction_tokens: Some(1),
                audio_tokens: None,
                reasoning_tokens: Some(2),
                rejected_prediction_tokens: Some(0),
            }),
        };
        let second_usage = CompletionUsage {
            prompt_tokens: 4,
            completion_tokens: 1,
            total_tokens: 5,
            prompt_tokens_details: Some(PromptTokensDetails {
                audio_tokens: Some(2),
                cached_tokens: Some(3),
            }),
            completion_tokens_details: Some(CompletionTokensDetails {
                accepted_prediction_tokens: Some(2),
                audio_tokens: Some(1),
                reasoning_tokens: None,
                rejected_prediction_tokens: Some(1),
            }),
        };

        let chunks = vec![
            Annotated::from_data(make_completion_chunk(
                "first",
                None,
                Some(continuous_usage.clone()),
            )),
            Annotated::from_data(make_completion_usage_chunk("cmpl-request-0", first_usage)),
            Annotated::from_data(make_completion_chunk("second", None, None)),
            Annotated::from_data(make_completion_usage_chunk("cmpl-request-1", second_usage)),
        ];

        let output: Vec<_> =
            aggregate_batch_completion_usage(futures::stream::iter(chunks), "request".to_string())
                .collect()
                .await;

        assert_eq!(output.len(), 3, "two content chunks and one usage chunk");
        assert_eq!(
            output[0]
                .data
                .as_ref()
                .and_then(|data| data.inner.usage.as_ref()),
            Some(&continuous_usage),
            "continuous usage must pass through unchanged",
        );

        let final_response = output[2].data.as_ref().expect("final data chunk");
        assert_eq!(final_response.inner.id, "cmpl-request");
        assert!(final_response.inner.choices.is_empty());
        let usage = final_response
            .inner
            .usage
            .as_ref()
            .expect("aggregate usage");
        assert_eq!(usage.prompt_tokens, 7);
        assert_eq!(usage.completion_tokens, 3);
        assert_eq!(usage.total_tokens, 10);
        let prompt_details = usage
            .prompt_tokens_details
            .as_ref()
            .expect("prompt token details");
        assert_eq!(prompt_details.audio_tokens, Some(3));
        assert_eq!(prompt_details.cached_tokens, Some(5));
        let completion_details = usage
            .completion_tokens_details
            .as_ref()
            .expect("completion token details");
        assert_eq!(completion_details.accepted_prediction_tokens, Some(3));
        assert_eq!(completion_details.audio_tokens, Some(1));
        assert_eq!(completion_details.reasoning_tokens, Some(2));
        assert_eq!(completion_details.rejected_prediction_tokens, Some(1));
    }

    #[tokio::test]
    async fn batch_completion_without_usage_is_unchanged() {
        let chunks = vec![Annotated::from_data(make_completion_chunk(
            "content", None, None,
        ))];

        let output: Vec<_> =
            aggregate_batch_completion_usage(futures::stream::iter(chunks), "request".to_string())
                .collect()
                .await;

        assert_eq!(output.len(), 1);
        let response = output[0].data.as_ref().expect("content chunk");
        assert_eq!(response.inner.id, "test");
        assert_eq!(response.inner.choices[0].text, "content");
        assert!(response.inner.usage.is_none());
    }
}