agent_engine/runtime/openai/
net.rs

1//! Error classification for the OpenAI-compat / Codex provider routes.
2//!
3//! Historically every error escaping `try_route` was blanket-wrapped in
4//! `RuntimeError::Config("openai provider: …")` — so a transient network
5//! blip against `chatgpt.com/backend-api/codex/responses` surfaced to the
6//! user as **"Config error"**, implying their configuration was broken
7//! (real incident: session 20260714-025948-3dab). This module classifies
8//! provider-route errors into honest categories:
9//!
10//! * transport failures (`reqwest::Error`) → `RuntimeError::ApiStatus` with
11//!   a host-aware humanized message that preserves the full cause chain
12//! * upstream HTTP status failures (`codex request failed: 503: …`) →
13//!   `RuntimeError::ApiStatus`
14//! * cancellation → `RuntimeError::Canceled`
15//! * everything else (missing keys, malformed tokens, …) → genuine
16//!   `RuntimeError::Config`, same prefix as before
17
18use crate::core::error::{error_chain_string, humanize_provider_network_error};
19use crate::RuntimeError;
20
21/// Boxed error type produced by the provider routes.
22pub type BoxedProviderError = Box<dyn std::error::Error + Send + Sync>;
23
24/// Classify an error escaping the OpenAI/Codex provider route into the
25/// right `RuntimeError` variant. Also logs the full cause chain at WARN —
26/// provider-route failures were previously invisible in the logs.
27///
28/// Provider-agnostic form: broker status labels are classified but not
29/// humanized. Prefer [`provider_error_to_runtime_for`] when the qualified
30/// model (and therefore the provider key) is known.
31pub fn provider_error_to_runtime(e: BoxedProviderError) -> RuntimeError {
32    provider_error_to_runtime_for("", e)
33}
34
35/// Like [`provider_error_to_runtime`], but `qualified_model` (e.g.
36/// `kimi-code/k3`) lets a vetted broker status label such as
37/// `provider request failed: 403 Forbidden [access_terminated_error]` be
38/// rendered as an actionable, provider-aware message. The label is one of
39/// OUR static constants (selected broker-side, never provider bytes) and the
40/// humanizer only ever returns our own text; the raw prefix message is kept
41/// for the log line.
42pub fn provider_error_to_runtime_for(qualified_model: &str, e: BoxedProviderError) -> RuntimeError {
43    // Transport-level failure (connect / timeout / mid-stream drop)?
44    if let Some(re) = find_reqwest_error(e.as_ref()) {
45        let msg = humanize_provider_network_error(re);
46        tracing::warn!(error = %error_chain_string(e.as_ref()), "provider transport error");
47        return RuntimeError::ApiStatus(msg);
48    }
49
50    let msg = error_chain_string(e.as_ref());
51
52    if msg.contains("operation canceled") || msg.contains("request canceled") {
53        return RuntimeError::Canceled;
54    }
55
56    // Upstream returned an HTTP error status or the Responses API reported a
57    // terminal stream failure — API failures, not configuration errors.
58    if msg.starts_with("codex request failed:")
59        || msg.starts_with("openai request failed:")
60        || is_responses_terminal_failure_message(&msg)
61    {
62        tracing::warn!(error = %msg, "provider API error");
63        if let Some(human) = humanize_broker_status_message(qualified_model, &msg) {
64            return RuntimeError::ApiStatus(human);
65        }
66        return RuntimeError::ApiStatus(msg);
67    }
68
69    tracing::warn!(error = %msg, "provider config error");
70    RuntimeError::Config(format!("openai provider: {msg}"))
71}
72
73/// Render a broker status failure carrying a vetted label as an actionable
74/// message. `None` when the message has no label, the label is unknown to
75/// the humanizer, or the provider key cannot be derived from the model.
76fn humanize_broker_status_message(qualified_model: &str, msg: &str) -> Option<String> {
77    let provider = qualified_model.split_once('/')?.0;
78    if provider.is_empty() {
79        return None;
80    }
81    let status = crate::runtime::trace::openai::broker_error_status(msg)?;
82    let label = broker_error_label(msg)?;
83    crate::core::error::humanize_proxy_status_error(provider, status, Some(label))
84}
85
86/// Admit only the complete broker-authored HTTP status envelope. Other broker
87/// errors (policy/config/credentials/transport ambiguity) keep their category.
88pub(super) fn normalize_broker_http_error(message: String) -> String {
89    let valid = message
90        .strip_prefix("broker transport error: provider request failed: ")
91        .is_some_and(|rest| {
92            let Some((status, tail)) = rest.split_once(' ') else {
93                return false;
94            };
95            if status.len() != 3 || !status.bytes().all(|b| b.is_ascii_digit()) {
96                return false;
97            }
98            let Ok(status) = status
99                .parse::<u16>()
100                .ok()
101                .and_then(|n| reqwest::StatusCode::from_u16(n).ok())
102                .ok_or(())
103            else {
104                return false;
105            };
106            if !status.is_client_error() && !status.is_server_error() {
107                return false;
108            }
109            let Some(reason) = status.canonical_reason() else {
110                return false;
111            };
112            if tail == reason {
113                return true;
114            }
115            tail.strip_prefix(reason)
116                .and_then(|r| r.strip_prefix(" ["))
117                .and_then(|r| r.strip_suffix(']'))
118                .is_some_and(|label| crate::core::error::VETTED_PROXY_ERROR_TYPES.contains(&label))
119        });
120    if valid {
121        format!("openai request failed: {message}")
122    } else {
123        message
124    }
125}
126
127/// Extract the `[label]` the broker appends after the canonical reason
128/// phrase (`provider request failed: 403 Forbidden [access_terminated_error]`).
129/// Only identifier-shaped labels are accepted; anything else is `None`.
130pub(crate) fn broker_error_label(msg: &str) -> Option<&str> {
131    const MARKER: &str = "provider request failed: ";
132    let rest = &msg[msg.find(MARKER)? + MARKER.len()..];
133    let open = rest.find(" [")?;
134    let after = &rest[open + 2..];
135    let close = after.find(']')?;
136    let label = &after[..close];
137    (!label.is_empty()
138        && label
139            .bytes()
140            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'))
141    .then_some(label)
142}
143
144/// Responses-API terminal stream failures are `"{label}{suffix}"` where the
145/// label is a provider display name ("Codex", "xAI") and the suffix is one of
146/// the static templates in `stream.rs`. Match label-independently on the
147/// suffix so every provider label classifies as an API failure.
148fn is_responses_terminal_failure_message(msg: &str) -> bool {
149    use super::stream::{
150        RESPONSES_AUTH_SUFFIX, RESPONSES_CAPACITY_SUFFIX, RESPONSES_CONTEXT_SUFFIX,
151        RESPONSES_EMPTY_SUFFIX, RESPONSES_FAILED_SUFFIX, RESPONSES_INCOMPLETE_SUFFIX,
152        RESPONSES_MISSING_TERMINAL_SUFFIX, RESPONSES_QUOTA_SUFFIX,
153    };
154    const SUFFIXES: &[&str] = &[
155        RESPONSES_AUTH_SUFFIX,
156        RESPONSES_QUOTA_SUFFIX,
157        RESPONSES_FAILED_SUFFIX,
158        RESPONSES_CAPACITY_SUFFIX,
159        RESPONSES_CONTEXT_SUFFIX,
160        RESPONSES_INCOMPLETE_SUFFIX,
161        RESPONSES_EMPTY_SUFFIX,
162        RESPONSES_MISSING_TERMINAL_SUFFIX,
163    ];
164    SUFFIXES.iter().any(|suffix| {
165        msg.strip_suffix(suffix)
166            .is_some_and(|label| !label.is_empty() && !label.contains(char::is_whitespace))
167    })
168}
169
170/// Walk the boxed error and its source chain looking for a `reqwest::Error`.
171///
172/// `send().await?` boxes the `reqwest::Error` directly, but helpers may wrap
173/// it another level deep — check the whole chain, not just the top.
174fn find_reqwest_error<'a>(e: &'a (dyn std::error::Error + 'static)) -> Option<&'a reqwest::Error> {
175    let mut current: Option<&(dyn std::error::Error + 'static)> = Some(e);
176    while let Some(err) = current {
177        if let Some(re) = err.downcast_ref::<reqwest::Error>() {
178            return Some(re);
179        }
180        current = err.source();
181    }
182    None
183}
184
185/// Strip the provider response-body snippet from a broker proxy error.
186///
187/// `LocalBroker::proxy_stream` flattens an upstream HTTP failure into
188/// `… provider request failed: {status}: {body snippet}`. The snippet is
189/// provider-controlled and may echo the full request (spec §5.1) — a byte
190/// bound is not redaction. Keep everything up to and including the status
191/// (canonical reason phrase, no `:`), drop the snippet. Messages without the
192/// marker pass through unchanged: every other `BrokerError` variant carries
193/// broker-authored, secret-free text, and transport-level reqwest errors are
194/// not response bodies.
195pub(crate) fn redact_provider_proxy_error(msg: &str) -> String {
196    const MARKER: &str = "provider request failed: ";
197    let Some(start) = msg.find(MARKER) else {
198        return msg.to_string();
199    };
200    let status_start = start + MARKER.len();
201    match msg[status_start..].find(':') {
202        Some(idx) => msg[..status_start + idx].to_string(),
203        None => msg.to_string(),
204    }
205}
206
207#[cfg(test)]
208mod tests {
209    #[test]
210    fn broker_http_envelope_retains_provider_error_provenance() {
211        use crate::extensions::session_driver::{classify_turn_error, Outcome};
212        for (tail, expected) in [
213            ("401 Unauthorized", Some("auth")),
214            ("402 Payment Required", Some("quota")),
215            ("429 Too Many Requests", Some("rate_limit")),
216            ("429 Too Many Requests [insufficient_quota]", Some("quota")),
217            ("503 Service Unavailable", Some("transient")),
218            ("403 Forbidden", None),
219        ] {
220            let message = super::normalize_broker_http_error(format!(
221                "broker transport error: provider request failed: {tail}"
222            ));
223            assert!(message.starts_with("openai request failed: "));
224            let runtime = super::provider_error_to_runtime_for("xai-auth/grok-4.6", message.into());
225            assert!(matches!(runtime, crate::RuntimeError::ApiStatus(_)));
226            let result =
227                classify_turn_error(&crate::runtime::helpers::turn_error_for(&runtime, "test"));
228            match expected {
229                Some(kind) => assert_eq!(result, (Outcome::ProviderError, kind.into())),
230                None => assert_eq!(result.0, Outcome::Blocked),
231            }
232        }
233        for message in [
234            "broker transport error: private config problem",
235            "broker transport error: provider request failed: 401 untrusted reason",
236            "broker transport error: provider request failed: 401 Unauthorized: private body",
237            "broker transport error: provider request failed: 429 Too Many Requests [unlisted_error]",
238            "quoted broker transport error: provider request failed: 503 Service Unavailable",
239        ] {
240            assert_eq!(super::normalize_broker_http_error(message.into()), message);
241        }
242    }
243    use super::*;
244
245    async fn connect_error() -> reqwest::Error {
246        // Bind then drop → guaranteed-refused port.
247        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
248        let addr = listener.local_addr().unwrap();
249        drop(listener);
250        reqwest::Client::new()
251            .post(format!("http://{addr}/codex/responses"))
252            .send()
253            .await
254            .expect_err("must fail to connect")
255    }
256
257    #[tokio::test]
258    async fn transport_error_is_api_status_not_config() {
259        let err = connect_error().await;
260        let runtime_err = provider_error_to_runtime(Box::new(err));
261        assert!(
262            matches!(runtime_err, RuntimeError::ApiStatus(_)),
263            "transport failure must classify as ApiStatus, got: {runtime_err:?}"
264        );
265        let display = runtime_err.to_string();
266        assert!(
267            !display.starts_with("Config error"),
268            "transient network failure must not be presented as a config problem: {display}"
269        );
270        assert!(
271            display.contains("127.0.0.1"),
272            "must name the host: {display}"
273        );
274    }
275
276    #[tokio::test]
277    async fn transport_error_preserves_cause_chain() {
278        let err = connect_error().await;
279        let display = provider_error_to_runtime(Box::new(err)).to_string();
280        // The top-level reqwest Display alone says only "error sending
281        // request" — the classified message must keep the underlying cause.
282        assert!(
283            display.contains("refused") || display.contains("connect"),
284            "underlying cause must survive classification: {display}"
285        );
286    }
287
288    #[test]
289    fn upstream_status_error_is_api_status() {
290        let e: BoxedProviderError = "codex request failed: 503: upstream sad".into();
291        let runtime_err = provider_error_to_runtime(e);
292        assert!(
293            matches!(runtime_err, RuntimeError::ApiStatus(_)),
294            "HTTP status failure must classify as ApiStatus, got: {runtime_err:?}"
295        );
296        assert!(runtime_err.to_string().contains("503"));
297    }
298
299    #[test]
300    fn openai_status_error_is_api_status() {
301        let e: BoxedProviderError = "openai request failed: 429: slow down".into();
302        assert!(matches!(
303            provider_error_to_runtime(e),
304            RuntimeError::ApiStatus(_)
305        ));
306    }
307
308    #[test]
309    fn responses_terminal_errors_are_api_status_not_config() {
310        for msg in [
311            "Codex response failed in stream. Provider error details withheld because they can echo request content.",
312            "Codex response was incomplete. Retry the request or reduce the requested output/context size.",
313            "Codex completed without text or tool output. Retry the request.",
314            "Codex response stream ended without a terminal event. Retry the request.",
315            "Codex rejected the request: the conversation exceeds this model's context window. Run /compact or start a fresh session to continue.",
316            // xAI-labelled variants (xai-auth Responses route).
317            "xAI response failed in stream. Provider error details withheld because they can echo request content.",
318            "xAI reports the model is at capacity (retries exhausted). Try again in a few minutes or switch models with /model.",
319            "xAI completed without text or tool output. Retry the request.",
320            "xAI response stream ended without a terminal event. Retry the request.",
321            "xAI response was incomplete. Retry the request or reduce the requested output/context size.",
322        ] {
323            let err = provider_error_to_runtime(msg.into());
324            assert!(
325                matches!(err, RuntimeError::ApiStatus(_)),
326                "Responses terminal must classify as ApiStatus: {err:?}"
327            );
328            assert!(!err.to_string().starts_with("Config error"));
329        }
330    }
331
332    /// A message that merely mentions a suffix mid-sentence (or carries a
333    /// non-label prefix) must not be mistaken for a Responses terminal.
334    #[test]
335    fn responses_terminal_matcher_requires_bare_label_prefix() {
336        assert!(!is_responses_terminal_failure_message(
337            "No API key for 'xai'. response failed in stream."
338        ));
339        assert!(!is_responses_terminal_failure_message(
340            " response failed in stream. Provider error details withheld because they can echo request content."
341        ));
342        assert!(is_responses_terminal_failure_message(
343            "xAI reports the model is at capacity (retries exhausted). Try again in a few minutes or switch models with /model."
344        ));
345    }
346
347    #[test]
348    fn cancellation_is_canceled() {
349        let e: BoxedProviderError = "operation canceled".into();
350        assert!(matches!(
351            provider_error_to_runtime(e),
352            RuntimeError::Canceled
353        ));
354        let e: BoxedProviderError = "request canceled".into();
355        assert!(matches!(
356            provider_error_to_runtime(e),
357            RuntimeError::Canceled
358        ));
359    }
360
361    #[test]
362    fn redact_provider_proxy_error_drops_body_snippet_keeps_status() {
363        let msg = "gemini stream error: broker transport error: provider request failed: \
364                   429 Too Many Requests: {\"error\":{\"message\":\"ECHOED:secret\"}}";
365        let redacted = redact_provider_proxy_error(msg);
366        assert_eq!(
367            redacted,
368            "gemini stream error: broker transport error: provider request failed: \
369             429 Too Many Requests"
370        );
371    }
372
373    #[test]
374    fn redact_provider_proxy_error_passes_through_non_proxy_messages() {
375        for msg in [
376            "broker transport error: connection reset",
377            "no credential configured for 'groq'. Run `synaps login` to add one.",
378            "request canceled",
379        ] {
380            assert_eq!(redact_provider_proxy_error(msg), msg);
381        }
382    }
383
384    #[test]
385    fn genuine_config_problems_stay_config_with_original_prefix() {
386        let e: BoxedProviderError =
387            "No API key for 'groq'. Set provider.groq in ~/.synaps-cli/config or the corresponding env var."
388                .into();
389        let runtime_err = provider_error_to_runtime(e);
390        assert!(
391            matches!(runtime_err, RuntimeError::Config(_)),
392            "missing key IS a config problem, got: {runtime_err:?}"
393        );
394        assert!(
395            runtime_err
396                .to_string()
397                .starts_with("Config error: openai provider: "),
398            "must keep the historical prefix for genuine config errors: {runtime_err}"
399        );
400    }
401
402    /// A broker status failure carrying a vetted label is humanized into an
403    /// actionable, provider-aware message when the qualified model is known
404    /// — and never echoes anything but our own text.
405    #[test]
406    fn vetted_kimi_quota_label_is_humanized_for_known_provider() {
407        let raw = "openai request failed: broker transport error: provider request failed: 403 Forbidden [access_terminated_error]";
408        let err: BoxedProviderError = raw.into();
409        let runtime_err = provider_error_to_runtime_for("kimi-code/k3", err);
410        let RuntimeError::ApiStatus(msg) = runtime_err else {
411            panic!("expected ApiStatus, got {runtime_err:?}");
412        };
413        assert!(
414            msg.starts_with("kimi-code: usage quota exhausted (HTTP 403)"),
415            "{msg}"
416        );
417        assert!(msg.contains("weekly"), "{msg}");
418        assert!(
419            !msg.contains("login"),
420            "quota must not read as an auth failure: {msg}"
421        );
422        assert!(
423            !msg.contains("[access_terminated_error]"),
424            "label must be rendered, not leaked: {msg}"
425        );
426    }
427
428    /// Without a provider key (legacy entry point) or without a label the
429    /// raw, status-only prefix message is preserved verbatim.
430    #[test]
431    fn broker_status_without_label_or_provider_keeps_raw_message() {
432        let labelled = "openai request failed: broker transport error: provider request failed: 403 Forbidden [access_terminated_error]";
433        match provider_error_to_runtime(labelled.into()) {
434            RuntimeError::ApiStatus(msg) => assert_eq!(msg, labelled),
435            other => panic!("expected ApiStatus, got {other:?}"),
436        }
437        let plain =
438            "openai request failed: broker transport error: provider request failed: 403 Forbidden";
439        match provider_error_to_runtime_for("kimi-code/k3", plain.into()) {
440            RuntimeError::ApiStatus(msg) => assert_eq!(msg, plain),
441            other => panic!("expected ApiStatus, got {other:?}"),
442        }
443        // Unknown (status, label) combos also fall through unchanged.
444        let unknown =
445            "openai request failed: provider request failed: 418 I'm a teapot [server_error]";
446        match provider_error_to_runtime_for("groq/llama", unknown.into()) {
447            RuntimeError::ApiStatus(msg) => assert_eq!(msg, unknown),
448            other => panic!("expected ApiStatus, got {other:?}"),
449        }
450    }
451
452    #[test]
453    fn broker_error_label_accepts_only_identifier_shaped_labels() {
454        assert_eq!(
455            broker_error_label(
456                "x: provider request failed: 403 Forbidden [access_terminated_error]"
457            ),
458            Some("access_terminated_error")
459        );
460        assert_eq!(
461            broker_error_label("provider request failed: 403 Forbidden"),
462            None
463        );
464        assert_eq!(
465            broker_error_label("provider request failed: 403 Forbidden []"),
466            None
467        );
468        assert_eq!(
469            broker_error_label("provider request failed: 403 Forbidden [<script>ECHO]"),
470            None
471        );
472        assert_eq!(
473            broker_error_label("provider request failed: 403 Forbidden [Mixed-Case]"),
474            None
475        );
476        assert_eq!(
477            broker_error_label("no marker here [access_terminated_error]"),
478            None
479        );
480    }
481}
agent_engine/runtime/openai/net.rs

agent_engine/runtime/openai/
net.rs
