polyc-tools 2026.9.0

//! Typed connector-error taxonomy and the retry policy keyed on it.
//!
//! A remote tool connector fails in three ways that call for three different
//! responses, so the failure is classified into a [`ConnectorErrorKind`] and the
//! caller acts on the kind rather than parsing a string. The taxonomy mirrors
//! the provider path's `LlmErrorKind` (`polyc_llm::LlmErrorKind`): a small,
//! stable set naming only the distinctions a caller acts on.
//!
//! - [`ConnectorErrorKind::Transport`] — the connector was unreachable or slow.
//!   Worth a brief, jittered retry inside a bounded budget.
//! - [`ConnectorErrorKind::Auth`] — the connector rejected the request's
//!   credentials. Never retried; surfaced typed so the dial path can route it to
//!   credential renewal.
//! - [`ConnectorErrorKind::Application`] — the connector answered but the tool
//!   call itself failed. Handed to the model as a tool result, untried.
//! - [`ConnectorErrorKind::Config`] — the deployment's own connector
//!   configuration is wrong (a malformed label, an unparseable resource URI).
//!   Never retried, and never routed to credential renewal — the fix is a
//!   config change, so it is kept distinct from [`ConnectorErrorKind::Auth`].
//!
//! [`CallRetryPolicy`] carries the retry envelope. Its budget is a small
//! fraction of the harness step loop's per-turn wall clock, so retrying a flaky
//! connector never blows the turn deadline.

use std::time::Duration;

use rmcp::{
    model::CallToolResult, service::ServiceError,
    transport::streamable_http_client::StreamableHttpError,
};
use serde_json::json;

use crate::mcp_client::McpClientError;

/// A coarse, retry-relevant classification of a connector call or dial failure.
///
/// Deliberately small and stable — it names only the distinctions a caller acts
/// on (retry vs. surface-for-rotation vs. return-to-model), not a full transport
/// taxonomy. Mirrors the provider path's `LlmErrorKind`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConnectorErrorKind {
    /// The connector could not be reached, or did not answer in time — a refused
    /// or reset connection, DNS failure, read/connect timeout, or a transient
    /// upstream `5xx`. Retryable inside a bounded budget: the same request to the
    /// same connector may well succeed a moment later.
    Transport,
    /// The connector rejected the request's credentials (HTTP `401`/`403`, on the
    /// dial or on a call). Terminal — retrying with the same token never helps —
    /// so it is surfaced typed for the caller to route to credential renewal
    /// rather than retried.
    Auth,
    /// The connector answered, but the tool call itself failed — a JSON-RPC error
    /// response, or a tool result flagged `isError`. This is the connector
    /// working as designed and reporting a per-call problem, so it goes to the
    /// model as an ordinary (failed) tool result, untried.
    Application,
    /// The deployment's own connector configuration is invalid — a malformed
    /// connector label or an unparseable resource URI. Terminal, but NOT a
    /// credential problem: routing it to credential renewal would mask the
    /// misconfiguration, so it carries its own kind.
    Config,
}

impl ConnectorErrorKind {
    /// Whether a failure of this kind is worth retrying. Only
    /// [`Self::Transport`] is: an [`Self::Auth`] rejection needs new
    /// credentials, an [`Self::Application`] error is the connector's own
    /// considered answer, and a [`Self::Config`] error needs a deployment
    /// fix — none of those are improved by trying again.
    #[must_use]
    pub const fn retryable(self) -> bool {
        matches!(self, Self::Transport)
    }

    /// A stable machine token naming this kind in the model-facing error object.
    ///
    /// Kept stable across releases so a consumer (or the model) can branch on the
    /// `kind` field of the returned JSON without matching on prose.
    #[must_use]
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Transport => "transport_unreachable",
            Self::Auth => "auth_rejected",
            Self::Application => "application_error",
            Self::Config => "config_error",
        }
    }
}

/// The retry envelope for a single connector tool call, keyed on
/// [`ConnectorErrorKind`] by the executor that owns it.
///
/// Only [`ConnectorErrorKind::Transport`] failures are retried, and only within
/// [`Self::budget`] — a total wall-clock ceiling across every attempt for one
/// call. Each retry waits an equal-jittered exponential backoff (see the
/// provider path's `polyc_llm::retry::backoff_delay`), so a fleet of pods
/// dialing the same recovering connector does not thunder in lockstep.
///
/// # Shared clock
///
/// [`Self::budget`] is deliberately a small fraction of the harness step loop's
/// per-turn wall clock (`HARNESS_TURN_DEADLINE`, ten minutes): a turn composes
/// its connectors and runs several tool round-trips inside that one deadline, so
/// no single call may spend more than a moment recovering a flaky connector or
/// the whole turn would starve. Tune it through [`Self::DEFAULT`], never with an
/// inline literal at a call site.
///
/// [`Self::call_timeout`] bounds a single attempt rather than the retry
/// sequence — see its own doc comment for why that budget is generous
/// (seconds, not milliseconds) despite [`Self::budget`] being tight.
#[derive(Debug, Clone, Copy)]
pub struct CallRetryPolicy {
    /// Total wall-clock budget across every attempt of ONE call. Once the next
    /// backoff would cross this, the last failure is surfaced instead.
    pub budget: Duration,
    /// Base backoff delay; the nth retry waits about `base_delay * 2^n`,
    /// equal-jittered and capped at [`Self::max_delay`].
    pub base_delay: Duration,
    /// Ceiling on any single backoff wait, so a late attempt in a large budget
    /// still spaces sanely rather than sleeping for the whole remainder.
    pub max_delay: Duration,
    /// Wall-clock bound on a single `tools/call` attempt.
    ///
    /// A remote tool call is an ordinary bounded RPC, not a human-in-the-loop
    /// wait — nothing on this path is waiting on a person to click a button,
    /// the way an approval pause is, so there is no case for letting it run
    /// generous-to-the-turn-budget. It is instead sized for the slowest
    /// *legitimate* connector operation (a heavier search, a file fetch, a
    /// multi-step remote action) while staying a small fraction of the
    /// control plane's own per-turn deadline (`TURN_DEADLINE` in
    /// `polyc-control-plane`, which is minutes rather than seconds and moves
    /// with the cold-start chain): thirty seconds leaves the rest of that budget
    /// for the dial, the model round-trips, and any other tool calls the same
    /// turn makes, so one stalled connector degrades to a typed failure the
    /// model can act on instead of parking delta forwarding until the whole
    /// turn dies as `deadline_exceeded`.
    pub call_timeout: Duration,
}

impl CallRetryPolicy {
    /// The shared default envelope: a two-second retry budget — long enough to
    /// ride out a brief connector blip, short enough to stay a rounding error
    /// against the ten-minute harness turn deadline — with a 100 ms base
    /// backoff capped at 500 ms, and a 30-second bound on any one attempt (see
    /// [`Self::call_timeout`]).
    pub const DEFAULT: Self = Self {
        budget: Duration::from_secs(2),
        base_delay: Duration::from_millis(100),
        max_delay: Duration::from_millis(500),
        call_timeout: Duration::from_secs(30),
    };
}

impl Default for CallRetryPolicy {
    fn default() -> Self {
        Self::DEFAULT
    }
}

/// Classify a live tool-call failure ([`rmcp::service::ServiceError`]) into the
/// shared [`ConnectorErrorKind`].
///
/// - A JSON-RPC error response ([`ServiceError::McpError`]) is the connector's
///   own considered answer that the call failed — [`ConnectorErrorKind::Application`].
/// - [`ServiceError::InputRequiredRoundsExceeded`] (SEP-2322 MRTR) means the
///   connector kept answering `input_required` past the round cap without ever
///   producing a final result — a misbehaving server, not a reachability blip:
///   retrying the same bounded round-trip loop again would not help, so this
///   is [`ConnectorErrorKind::Application`] too, not [`ConnectorErrorKind::Transport`].
/// - A request timeout is a reachability problem — [`ConnectorErrorKind::Transport`].
/// - A send failure is inspected for an auth signal: an HTTP `401`/`403` —
///   surfaced by rmcp as an auth-required / insufficient-scope transport error,
///   or a bare reqwest status — is [`ConnectorErrorKind::Auth`]; everything else
///   (refused/reset/`5xx`/parse) is [`ConnectorErrorKind::Transport`].
pub(crate) fn classify_call_error(err: &ServiceError) -> ConnectorErrorKind {
    match err {
        ServiceError::McpError(_) | ServiceError::InputRequiredRoundsExceeded { .. } => {
            ConnectorErrorKind::Application
        }
        ServiceError::TransportSend(dyn_err) => {
            if transport_source_is_auth(dyn_err.error.as_ref()) {
                ConnectorErrorKind::Auth
            } else {
                ConnectorErrorKind::Transport
            }
        }
        // A timeout and every other transport-family variant (closed, cancelled,
        // unexpected response) are reachability failures worth a bounded retry.
        _ => ConnectorErrorKind::Transport,
    }
}

/// Whether an error's source chain carries an HTTP `401`/`403` auth signal.
///
/// rmcp surfaces a `401` with a `WWW-Authenticate` header as
/// [`StreamableHttpError::AuthRequired`] and a `403` as
/// [`StreamableHttpError::InsufficientScope`]; a status without that header
/// falls through to a reqwest client error carrying the status. The source
/// chain is walked so the signal is found regardless of how the transport
/// wraps it.
pub(crate) fn transport_source_is_auth(err: &(dyn std::error::Error + 'static)) -> bool {
    let mut current: Option<&(dyn std::error::Error + 'static)> = Some(err);
    while let Some(e) = current {
        if let Some(se) = e.downcast_ref::<StreamableHttpError<reqwest::Error>>() {
            match se {
                StreamableHttpError::AuthRequired(_)
                | StreamableHttpError::InsufficientScope(_) => {
                    return true;
                }
                StreamableHttpError::Client(re) if reqwest_is_auth(re) => return true,
                _ => {}
            }
        }
        if let Some(re) = e.downcast_ref::<reqwest::Error>()
            && reqwest_is_auth(re)
        {
            return true;
        }
        current = e.source();
    }
    false
}

/// Whether a reqwest error carries an HTTP `401`/`403` status.
fn reqwest_is_auth(err: &reqwest::Error) -> bool {
    err.status().is_some_and(|s| {
        s == reqwest::StatusCode::UNAUTHORIZED || s == reqwest::StatusCode::FORBIDDEN
    })
}

/// Map a handshake (`.serve`) failure onto [`McpClientError`], classifying a
/// rejected credential as [`McpClientError::AuthRejected`] (legible for
/// credential renewal) rather than a generic [`McpClientError::Init`].
pub(crate) fn dial_error<E: std::error::Error + 'static>(err: E) -> McpClientError {
    if transport_source_is_auth(&err) {
        McpClientError::AuthRejected(err.to_string())
    } else {
        McpClientError::Init(err.to_string())
    }
}

/// The success payload for a completed call: structured output verbatim, else
/// concatenated text content parsed as JSON when it parses, else a wrapped
/// string — the shape the agent loop folds straight into the next turn.
pub(crate) fn success_json(result: CallToolResult) -> String {
    if let Some(structured) = result.structured_content {
        structured.to_string()
    } else {
        let text: String = result
            .content
            .iter()
            .filter_map(|c| c.as_text().map(|t| t.text.clone()))
            .collect();
        if text.is_empty() {
            json!({ "ok": true }).to_string()
        } else {
            serde_json::from_str::<serde_json::Value>(&text)
                .map_or_else(|_| json!({ "result": text }).to_string(), |v| v.to_string())
        }
    }
}

/// The connector's own message for a result flagged `isError` — its structured
/// content, else its text, else a plain default.
pub(crate) fn result_message(result: &CallToolResult) -> String {
    if let Some(structured) = &result.structured_content {
        return structured.to_string();
    }
    let text: String = result
        .content
        .iter()
        .filter_map(|c| c.as_text().map(|t| t.text.clone()))
        .collect();
    if text.is_empty() {
        "The tool reported an error.".to_owned()
    } else {
        text
    }
}

/// The shared, stable sentence for a [`ConnectorErrorKind::Transport`] failure.
///
/// Covers unreachable, reset, or (via [`polyc_agent::ToolExecutor::execute`]'s
/// per-call timeout, as implemented by [`crate::mcp_client::McpToolSource`])
/// simply never answering in time. One helper so the wording never drifts
/// between the live-call path (`call_error_message`), the lazy dial path
/// ([`dial_failure_message`]), and a call-timeout, which has no
/// [`ServiceError`] to classify from.
///
/// The wording is load-bearing for #635: it says the tool is *temporarily*
/// unavailable and recovers on its own, so the model (and a reader of the
/// transcript) treats the outage as a blip to ride out — not a missing tool.
#[must_use]
pub const fn transport_failure_message() -> &'static str {
    "The tool is temporarily unavailable: its connector could not be reached. \
     Try again in a moment."
}

/// The plain, model-facing message for a terminal call failure of `kind`. A
/// transport or auth failure gets a stable, jargon-free sentence (never the raw
/// transport dump); an application error carries the connector's own message.
pub(crate) fn call_error_message(kind: ConnectorErrorKind, err: &ServiceError) -> String {
    match kind {
        ConnectorErrorKind::Transport => transport_failure_message().to_owned(),
        ConnectorErrorKind::Auth => {
            "The tool's connector rejected the request. Its access needs to be renewed.".to_owned()
        }
        ConnectorErrorKind::Config => {
            "The tool's connector is misconfigured. Its setup needs to be corrected.".to_owned()
        }
        ConnectorErrorKind::Application => match err {
            ServiceError::McpError(data) => data.message.to_string(),
            other => other.to_string(),
        },
    }
}

/// The plain, model-facing message for a failed connector DIAL of `kind`.
///
/// The lazy-`execute` counterpart of `call_error_message`, which needs a
/// live [`ServiceError`] it does not have here. A dial only ever fails
/// [`ConnectorErrorKind::Transport`] (unreachable), [`ConnectorErrorKind::Auth`]
/// (rejected credentials), or [`ConnectorErrorKind::Config`] (bad label / URI);
/// an [`ConnectorErrorKind::Application`] error cannot arise from a handshake, so
/// it falls back to the transport sentence.
#[must_use]
pub fn dial_failure_message(kind: ConnectorErrorKind) -> String {
    match kind {
        ConnectorErrorKind::Auth => {
            "The tool's connector rejected the request. Its access needs to be renewed.".to_owned()
        }
        ConnectorErrorKind::Config => {
            "The tool's connector is misconfigured. Its setup needs to be corrected.".to_owned()
        }
        ConnectorErrorKind::Transport | ConnectorErrorKind::Application => {
            transport_failure_message().to_owned()
        }
    }
}

/// The typed, stable failure object handed to the model: a plain `error`
/// sentence plus the machine-stable `kind` token, in place of an ad-hoc string.
#[must_use]
pub fn failure_json(kind: ConnectorErrorKind, message: &str) -> String {
    json!({ "error": message, "kind": kind.as_str() }).to_string()
}

#[cfg(test)]
mod tests {
    #![allow(clippy::pedantic, clippy::nursery, missing_docs)]
    use super::*;

    #[test]
    fn only_transport_is_retryable() {
        // The retry matrix: transport failures retry, credential and application
        // failures never do (new creds / the connector's own answer, not a blip).
        assert!(ConnectorErrorKind::Transport.retryable());
        assert!(!ConnectorErrorKind::Auth.retryable());
        assert!(!ConnectorErrorKind::Application.retryable());
        assert!(!ConnectorErrorKind::Config.retryable());
    }

    #[test]
    fn kind_tokens_are_stable() {
        // These strings are part of the model-facing error object's contract.
        assert_eq!(
            ConnectorErrorKind::Transport.as_str(),
            "transport_unreachable"
        );
        assert_eq!(ConnectorErrorKind::Auth.as_str(), "auth_rejected");
        assert_eq!(
            ConnectorErrorKind::Application.as_str(),
            "application_error"
        );
        assert_eq!(ConnectorErrorKind::Config.as_str(), "config_error");
    }

    #[test]
    fn default_budget_stays_well_under_the_turn_deadline() {
        // The shared-clock invariant: the per-call budget must be a small
        // fraction of the harness turn deadline (600s) so retries can't starve a
        // turn. A generous ceiling of one minute documents the intent as a test.
        let p = CallRetryPolicy::default();
        assert!(p.budget <= Duration::from_secs(60));
        assert!(p.base_delay <= p.max_delay);
        assert!(p.max_delay <= p.budget);
    }

    #[test]
    fn default_call_timeout_stays_a_fraction_of_the_control_plane_turn_deadline() {
        // The per-attempt bound is a different axis from `budget` (which gates
        // whether a failed attempt gets retried at all): it must stay a small
        // fraction of the control plane's own per-turn deadline
        // so one stalled connector call cannot alone hold a turn for most of
        // its budget, while still being generous enough (seconds, not
        // milliseconds) for a real remote tool operation to complete.
        let p = CallRetryPolicy::default();
        assert!(p.call_timeout >= Duration::from_secs(10));
        assert!(p.call_timeout <= Duration::from_secs(60));
    }
}