Skip to main content

orch/engine/runner/
fallback.rs

1//! Error recovery and fallback strategies.
2//!
3//! Extracted from `runner/mod.rs`. Handles the `Err(agent_err)` arm of the
4//! parse result: error classification, model failover, free-model fallback,
5//! and agent rerouting.
6
7use crate::store;
8use crate::store::TaskStore;
9use std::collections::HashSet;
10use std::sync::Arc;
11
12use super::{agents, response};
13use crate::cmd_cache;
14use crate::engine;
15use crate::engine::cooldown;
16use crate::engine::cooldown::CreditExhaustionReason;
17
18/// Summarizes a rate limit error by extracting key information from Claude's API retry JSON.
19///
20/// Looks for the last `{"type":"system","subtype":"api_retry",...}` line (or a partial
21/// fragment containing `"subtype":"api_retry"`) and extracts `error_status`, `attempt`,
22/// `retry_delay_ms` to return a compact summary.
23/// Returns a compact fallback if no api_retry JSON is found.
24///
25/// Exposed `pub(crate)` so the review pipeline (`engine::review`) reuses the same
26/// summariser as the task runner. Without this, review runs persisted raw
27/// api_retry JSON fragments into `task_runs.error` (issue #3149).
28pub(crate) fn summarize_rate_limit_error(raw_output: &str) -> String {
29    // Byte cap for the first-line fallback summary below
30    const FIRST_LINE_CAP: usize = 120;
31    // Find the last line containing "subtype":"api_retry"
32    // This handles both complete JSON objects and truncated fragments that begin
33    // mid-object (e.g. `,"subtype":"api_retry","attempt":5...`).
34    let last_api_retry_line = raw_output
35        .lines()
36        .rev()
37        .find(|line| line.contains("\"subtype\":\"api_retry\""));
38
39    if let Some(line) = last_api_retry_line {
40        return summarise_api_retry_line(line);
41    }
42
43    // No `"subtype":"api_retry"` marker but the payload may still be a truncated
44    // api_retry NDJSON fragment that lost the `"subtype"` key earlier in the
45    // stream. If any line contains the characteristic numeric fields
46    // (`error_status`, `attempt`, `retry_delay_ms`), summarise from that line
47    // instead of echoing the raw JSON into `task_runs.error` (issue #3149).
48    let api_retry_field_markers = ["\"error_status\":", "\"attempt\":", "\"retry_delay_ms\":"];
49    let field_line = raw_output
50        .lines()
51        .rev()
52        .find(|line| api_retry_field_markers.iter().any(|m| line.contains(m)));
53    if let Some(line) = field_line {
54        return summarise_api_retry_line(line);
55    }
56
57    // No api_retry JSON or field markers found — emit a compact first-line
58    // summary, never raw passthrough. Truncate to a tight byte cap so even
59    // long single-line provider payloads cannot bloat `task_runs.error`.
60    let first_line = raw_output.lines().next().unwrap_or(raw_output);
61    if first_line.len() > FIRST_LINE_CAP {
62        let cutoff = agents::truncate_at_char_boundary(first_line, FIRST_LINE_CAP);
63        let remaining = raw_output.len().saturating_sub(cutoff);
64        format!("{}... (+{} more chars)", &first_line[..cutoff], remaining)
65    } else {
66        first_line.to_string()
67    }
68}
69
70/// Extract `error_status`, `attempt`, and `retry_delay_ms` from a single
71/// `api_retry`-shaped JSON line (or fragment) and return a compact summary
72/// such as `"status=429 after 7 attempts (last delay 35s)"`. Falls back to
73/// `"status=… (api_retry fragment)"` when no numeric field is recoverable.
74fn summarise_api_retry_line(line: &str) -> String {
75    let error_status =
76        extract_json_field(line, "\"error_status\":").unwrap_or_else(|| "unknown".to_string());
77    let attempt = extract_json_field(line, "\"attempt\":").unwrap_or_else(|| "unknown".to_string());
78    let retry_delay_ms =
79        extract_json_field(line, "\"retry_delay_ms\":").unwrap_or_else(|| "0".to_string());
80
81    let has_field = error_status != "unknown"
82        || attempt != "unknown"
83        || (retry_delay_ms != "0" && retry_delay_ms != "unknown");
84
85    if has_field {
86        let retry_delay_s = retry_delay_ms.parse::<f64>().unwrap_or(0.0) / 1000.0;
87        #[expect(
88            clippy::cast_possible_truncation,
89            clippy::cast_sign_loss,
90            reason = "the saturating cast floors the delay to whole seconds"
91        )]
92        let retry_delay_s = retry_delay_s as u64;
93        format!("status={error_status} after {attempt} attempts (last delay {retry_delay_s}s)")
94    } else {
95        let status_hint = extract_json_field(line, "\"status\":")
96            .or_else(|| extract_json_field(line, "\"error\":"))
97            .unwrap_or_else(|| "429".to_string());
98        format!("status={status_hint} (api_retry fragment)")
99    }
100}
101
102/// Extracts a JSON field value as a string, handling both quoted strings and raw numbers.
103fn extract_json_field(json: &str, field: &str) -> Option<String> {
104    json.find(field).and_then(|start| {
105        let value_start = start + field.len();
106        let value_str = &json[value_start..];
107
108        // Handle quoted strings
109        if let Some(rest) = value_str.strip_prefix('"') {
110            let end = rest.find('"')?; // find closing quote
111            Some(rest[..end].to_string())
112        }
113        // Handle raw numbers (integers or floats)
114        else {
115            let end = value_str
116                .find(|c: char| !c.is_ascii_digit() && c != '.')
117                .unwrap_or(value_str.len());
118            let num_str = &value_str[..end];
119            num_str
120                .strip_suffix(',')
121                .map(ToString::to_string)
122                .or_else(|| Some(num_str.to_string()))
123        }
124    })
125}
126
127/// How `run()` should proceed after error handling.
128#[derive(Debug)]
129pub enum ErrorHandleResult {
130    /// Task was rerouted — `run()` should record metrics then return early.
131    EarlyReturn { status: String },
132    /// Normal failover applied (or task marked needs_review) — proceed to cleanup + metrics.
133    ///
134    /// `error` is the current run's attributed error message (e.g. "kimi timed out after 1801s").
135    /// Callers should pass this as `error_override` to `build_run_audit` so that stale
136    /// `last_error` values from a previous agent's run are not mis-attributed.
137    Continue { status: String, error: String },
138}
139
140/// Try to reroute the task to an untried free model.
141///
142/// Returns `Some(EarlyReturn { status: "new" })` if a free model was found and the
143/// task was rerouted, `None` otherwise (caller should fall through to normal failover).
144///
145/// - `exclude_models`: models to skip in addition to already-tried ones
146/// - `check_cooldowns`: whether to skip models currently in cooldown
147/// - `set_agent`: if `Some`, also overwrite the task's `agent` field
148/// - `apply_backoff`: whether to pace the reroute with `wait_for_fallback_backoff`
149/// - `reason_prefix`: prefix for the `last_error` message (free model name is appended)
150#[expect(
151    clippy::too_many_arguments,
152    reason = "each caller picks a different mix of reroute options"
153)]
154#[expect(
155    clippy::ref_option,
156    reason = "the store helpers it calls take the store as &Option"
157)]
158async fn try_free_model_reroute(
159    task_id: &str,
160    agent_name: &str,
161    agent_runner: &dyn agents::AgentRunner,
162    exclude_models: &[&str],
163    check_cooldowns: bool,
164    set_agent: Option<&str>,
165    apply_backoff: bool,
166    reason_prefix: &str,
167    store: &Option<Arc<TaskStore>>,
168    repo: &str,
169) -> Option<ErrorHandleResult> {
170    let free = agent_runner.free_models();
171    if free.is_empty() {
172        return None;
173    }
174    let tried_models: String = store::opt_store_get_task(store, repo, task_id)
175        .await
176        .map(|t| t.model_reroute_chain)
177        .unwrap_or_default();
178    let tried_set: HashSet<&str> = tried_models.split(',').filter(|s| !s.is_empty()).collect();
179    let next_free = free.iter().find(|m| {
180        !exclude_models.contains(&m.as_str())
181            && !tried_set.contains(m.as_str())
182            && (!check_cooldowns
183                || (!cooldown::is_agent_in_cooldown(agent_name)
184                    && !response::is_model_in_cooldown(agent_name, m)))
185    })?;
186    tracing::info!(task_id, model = %next_free, "{}", reason_prefix);
187    let new_tried = if tried_models.is_empty() {
188        next_free.clone()
189    } else {
190        format!("{tried_models},{next_free}")
191    };
192    let msg = format!("{reason_prefix} {next_free}");
193    let mut fields: Vec<(&str, serde_json::Value)> = vec![
194        ("model", serde_json::json!(next_free.clone())),
195        ("model_reroute_chain", serde_json::json!(new_tried)),
196        ("last_error", serde_json::json!(msg)),
197    ];
198    if let Some(agent) = set_agent {
199        fields.push(("agent", serde_json::json!(agent)));
200    }
201    if let Err(e) = store::store_set_result(store, repo, task_id, &fields).await {
202        tracing::warn!(task_id, err = %e, "try_free_model_reroute: failed to persist model update — not rerouting");
203        return None; // let caller fall through to normal failover
204    }
205    if apply_backoff {
206        response::wait_for_fallback_backoff(task_id, store, repo).await;
207    }
208    Some(ErrorHandleResult::EarlyReturn {
209        status: "new".to_string(),
210    })
211}
212
213/// Apply a vendor-provided future reset date as the model cooldown.
214///
215/// Skipped for credit exhaustion reasons other than billing cycle, which keep their agent-wide cooldowns.
216/// Returns true when the cooldown was set.
217async fn apply_vendor_reset_date(
218    agent_name: &str,
219    model_name: Option<&str>,
220    message: &str,
221) -> bool {
222    let Some(model) = model_name else {
223        return false;
224    };
225    match cooldown::detect_credit_exhaustion(message) {
226        None | Some(CreditExhaustionReason::BillingCycleExhausted) => {}
227        Some(_) => return false,
228    }
229    match cooldown::parse_retry_at(message) {
230        Some(until) if until > chrono::Utc::now().timestamp() => {
231            cooldown::set_model_cooldown_until(agent_name, model, until).await;
232            true
233        }
234        _ => false,
235    }
236}
237
238/// Handle an agent error: classify it, attempt recovery strategies, and update store.
239///
240/// Returns `Ok(ErrorHandleResult::EarlyReturn)` when the task was rerouted and
241/// `run()` should record metrics and return immediately (skipping tmux cleanup).
242/// Returns `Ok(ErrorHandleResult::Continue)` when `run()` should proceed to cleanup.
243#[expect(
244    clippy::too_many_arguments,
245    reason = "the error path needs the task, agent, model and store context together"
246)]
247pub async fn handle_error(
248    task_id: &str,
249    agent_err: &agents::AgentError,
250    agent_name: &str,
251    agent_runner: &dyn agents::AgentRunner,
252    model_name: Option<&str>,
253    complexity: Option<&str>,
254    new_attempts: u32,
255    store: &Option<Arc<TaskStore>>,
256    repo: &str,
257) -> anyhow::Result<ErrorHandleResult> {
258    tracing::warn!(
259        task_id,
260        agent = agent_name,
261        error = %agent_err,
262        "agent error, attempting recovery"
263    );
264
265    // Map AgentError to RetryableError for handle_failover(). credit_recorded
266    // skips the generic failure_count bump for already-recorded credit errors
267    let mut credit_recorded = false;
268    let (retryable, error_msg) = match agent_err {
269        agents::AgentError::RateLimit { message, .. } => {
270            let is_provider_returned_error = message
271                .to_ascii_lowercase()
272                .contains("provider returned error");
273            // Check for credit exhaustion - these require longer agent-wide cooldowns.
274            // For all other rate limits, record both the agent-level and model-level
275            // cooldowns immediately so the router can observe them before any concurrent
276            // tasks start their runs. handle_failover() also records the cooldown later,
277            // but setting it here means the in-memory map is updated right away — without
278            // this, concurrent dispatches that check cooldowns between now and
279            // handle_failover() will see no cooldown and waste another run against the
280            // same rate-limited model.
281            //
282            // Setting BOTH cooldowns is critical: the router's model selection
283            // (model_for_complexity) checks is_model_in_cooldown(agent, model) which
284            // uses the agent:model key, so a model-level cooldown is needed to prevent
285            // the same model from being re-selected even when the agent-level cooldown
286            // is set. Without the model-level cooldown, the model continues to be
287            // picked and fails repeatedly (issue #2153).
288            // Track whether the quota is model-scoped (BillingCycleExhausted + known model).
289            // When true we return ModelUnavailable so handle_failover skips the
290            // agent-level failure counter — the agent itself is healthy, only the
291            // specific model's plan quota was exhausted (issue #3382).
292            let mut billing_cycle_model_scoped = false;
293
294            if apply_vendor_reset_date(agent_name, model_name, message).await {
295                let reason = cooldown::detect_credit_exhaustion(message);
296                credit_recorded = reason.is_some();
297                billing_cycle_model_scoped =
298                    reason == Some(CreditExhaustionReason::BillingCycleExhausted);
299            } else if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
300                credit_recorded = true;
301                // For billing cycle exhaustion where we know the specific model, apply
302                // model-level persistent cooldown only. The quota is scoped to that
303                // provider sub-model (e.g. github-copilot/gpt-5-mini), not the entire
304                // agent account, so an agent-wide cooldown would incorrectly block
305                // unrelated models on the same agent.
306                if reason == CreditExhaustionReason::BillingCycleExhausted {
307                    if let Some(model) = model_name {
308                        cooldown::record_persistent_model_failure(agent_name, model).await;
309                        billing_cycle_model_scoped = true;
310                    } else {
311                        cooldown::record_credit_exhaustion(agent_name, reason).await;
312                    }
313                } else {
314                    cooldown::record_credit_exhaustion(agent_name, reason).await;
315                }
316            } else if let (Some(model), Some(window_secs)) =
317                (model_name, cooldown::parse_relative_usage_window(message))
318            {
319                // Vendor-advertised relative usage window (e.g. "weekly (7-day)
320                // usage limit") is authoritative — apply a model-specific cooldown
321                // for that exact duration instead of the generic exponential
322                // backoff. This is the same window-detection the `needs_review`
323                // call site (cooldown::record_rate_limit()) already applies; this
324                // in-task failover path bypassed it entirely, undoing the
325                // #3572/#3574 fix for the most common rate-limit path (issue #3580).
326                cooldown::set_model_cooldown(
327                    agent_name,
328                    model,
329                    window_secs,
330                    "rate_limit_usage_window",
331                )
332                .await;
333                tracing::info!(
334                    task_id,
335                    agent = agent_name,
336                    model,
337                    window_secs,
338                    "relative usage window detected: applying model-specific cooldown"
339                );
340            } else {
341                cooldown::record_agent_failure_with_message(agent_name, message).await;
342                // Also record the model-specific cooldown so model_for_complexity skips it.
343                if let Some(model) = model_name {
344                    if is_provider_returned_error {
345                        cooldown::record_transient_model_failure(agent_name, model).await;
346                    } else {
347                        response::record_model_failure(agent_name, model).await;
348                    }
349                }
350            }
351            // Use ModelUnavailable when billing cycle is exhausted for a known model so
352            // handle_failover does not call record_agent_failure_with_message (which would
353            // set an unwanted agent-level cooldown). The agent's other models remain available.
354            let retryable = if billing_cycle_model_scoped {
355                response::RetryableError::ModelUnavailable
356            } else {
357                response::RetryableError::UsageLimit
358            };
359            (
360                retryable,
361                format!(
362                    "{agent_name} rate limit: {}",
363                    summarize_rate_limit_error(message)
364                ),
365            )
366        }
367        agents::AgentError::Auth { message } => {
368            // Check for credit exhaustion first — those need longer agent-wide cooldowns.
369            // For plain auth failures (expired key, wrong token, etc.) record both the
370            // agent-level and model-level cooldowns so the router can observe them before
371            // concurrent tasks start another run against the same unavailable model.
372            let mut billing_cycle_model_scoped = false;
373
374            if apply_vendor_reset_date(agent_name, model_name, message).await {
375                let reason = cooldown::detect_credit_exhaustion(message);
376                credit_recorded = reason.is_some();
377                billing_cycle_model_scoped =
378                    reason == Some(CreditExhaustionReason::BillingCycleExhausted);
379            } else if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
380                credit_recorded = true;
381                // Billing cycle exhaustion scoped to a known model gets model-level
382                // persistent cooldown only — agent-wide cooldown would block unrelated models.
383                if reason == CreditExhaustionReason::BillingCycleExhausted {
384                    if let Some(model) = model_name {
385                        cooldown::record_persistent_model_failure(agent_name, model).await;
386                        billing_cycle_model_scoped = true;
387                    } else {
388                        cooldown::record_credit_exhaustion(agent_name, reason).await;
389                    }
390                } else {
391                    cooldown::record_credit_exhaustion(agent_name, reason).await;
392                }
393            } else {
394                cooldown::record_agent_failure_with_message(agent_name, message).await;
395                if let Some(model) = model_name {
396                    response::record_model_failure(agent_name, model).await;
397                }
398            }
399            // Use ModelUnavailable for model-scoped billing exhaustion so handle_failover
400            // does not set an agent-level failure cooldown (issue #3382).
401            let retryable = if billing_cycle_model_scoped {
402                response::RetryableError::ModelUnavailable
403            } else {
404                response::RetryableError::AuthError
405            };
406            (retryable, format!("{agent_name} auth error: {message}"))
407        }
408        agents::AgentError::Timeout { elapsed } => {
409            // A task time limit says the work was long, not that the model is
410            // unavailable. Retry without a model cooldown or rate_limits row
411            (
412                response::RetryableError::Timeout,
413                format!("{agent_name} timed out after {}s", elapsed.as_secs()),
414            )
415        }
416        agents::AgentError::Interrupted { message } => (
417            response::RetryableError::Interrupted,
418            format!("{agent_name} interrupted: {message}"),
419        ),
420        agents::AgentError::MissingTool { tool } => (
421            response::RetryableError::MissingTooling,
422            format!("missing tool: {tool}"),
423        ),
424        agents::AgentError::ModelUnavailable { model, message } => {
425            cooldown::record_model_unavailable(agent_name, model, message).await;
426
427            // Try next model before switching agent, unless the agent is cooled
428            // (an agent-wide cooldown must not be defeated by model substitution)
429            let agent_cooled = cooldown::is_agent_in_cooldown(agent_name);
430            let models = agent_runner.available_models();
431            let current_model = model_name.unwrap_or("");
432            let next_model = if agent_cooled {
433                None
434            } else {
435                models.iter().find(|m| {
436                    m.as_str() != current_model
437                        && m.as_str() != model
438                        && !response::is_model_in_cooldown(agent_name, m)
439                })
440            };
441            if let Some(next) = next_model {
442                tracing::info!(task_id, model = %next, "retrying with different model");
443                let msg = format!("model {model} unavailable, trying {next}");
444                if let Err(e) = store::store_set_result(
445                    store,
446                    repo,
447                    task_id,
448                    &[
449                        ("model", serde_json::json!(next.clone())),
450                        ("last_error", serde_json::json!(msg)),
451                    ],
452                )
453                .await
454                {
455                    tracing::warn!(task_id, err = %e, "failed to write model failover to store");
456                }
457                // Skip normal failover — we're retrying same agent with different model.
458                // Use "routed" (not "new") since agent/model are already stored;
459                // this avoids a redundant LLM re-routing cycle.
460                response::wait_for_fallback_backoff(task_id, store, repo).await;
461                return Ok(ErrorHandleResult::EarlyReturn {
462                    status: "routed".to_string(),
463                });
464            }
465
466            // No other standard models available — try free models for simple tasks
467            let is_simple = matches!(complexity, None | Some("simple"));
468            if is_simple
469                && let Some(result) = try_free_model_reroute(
470                    task_id,
471                    agent_name,
472                    agent_runner,
473                    &[current_model, model.as_str()],
474                    true,
475                    None,
476                    true,
477                    &format!("model {model} unavailable, trying free model"),
478                    store,
479                    repo,
480                )
481                .await
482            {
483                return Ok(result);
484            }
485
486            (
487                response::RetryableError::ModelUnavailable,
488                format!("model {model} unavailable"),
489            )
490        }
491        agents::AgentError::ContextOverflow { message } => {
492            // Context overflow is deterministic — the task's prompt/context is too large
493            // for this model's context window. Cycling through other agents would hit the
494            // same limit and waste API calls. Skip failover entirely and escalate
495            // immediately to needs_review so a human can intervene (e.g., split the task,
496            // reduce context, or manually assign a model with a larger context window).
497            let msg = format!("{agent_name} context overflow: {message}");
498            if let Err(e) = store::store_set_result(
499                store,
500                repo,
501                task_id,
502                &[("last_error", serde_json::json!(msg))],
503            )
504            .await
505            {
506                tracing::warn!(task_id, err = %e, "failed to write context_overflow last_error to store");
507            }
508            return Ok(ErrorHandleResult::EarlyReturn {
509                status: "needs_review".to_string(),
510            });
511        }
512        agents::AgentError::WaitingForInput { message } => {
513            // Requires human — skip failover, go straight to needs_review
514            let msg = format!("waiting for input: {message}");
515            if let Err(e) = store::store_set_result(
516                store,
517                repo,
518                task_id,
519                &[("last_error", serde_json::json!(msg))],
520            )
521            .await
522            {
523                tracing::warn!(task_id, err = %e, "failed to write waiting_for_input last_error to store");
524            }
525            return Ok(ErrorHandleResult::EarlyReturn {
526                status: "needs_review".to_string(),
527            });
528        }
529        agents::AgentError::PermissionDenied { message } => (
530            response::RetryableError::Failed,
531            format!("permission denied: {message}"),
532        ),
533        agents::AgentError::InvalidResponse { .. } => {
534            // Record model-specific cooldown so the same model that produced an
535            // unparseable response isn't immediately retried on the next task.
536            if let Some(model) = model_name {
537                response::record_model_failure(agent_name, model).await;
538            }
539            (
540                response::RetryableError::Failed,
541                format!("{agent_name} invalid response"),
542            )
543        }
544        agents::AgentError::AgentFailed { message } => {
545            // "Provider returned error" (opencode) and similar upstream failures.
546            // Record model+agent cooldowns so the router skips this model on retry,
547            // then route through handle_failover() to get proper reroute-chain
548            // tracking, exhaustion checks, and fallback pacing.
549            // Credit-shaped errors that escaped the Auth/RateLimit classifiers still
550            // need the credit cooldown and counter, not the generic 5 min backoff.
551            if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
552                credit_recorded = true;
553                match model_name {
554                    Some(model) if reason == CreditExhaustionReason::BillingCycleExhausted => {
555                        cooldown::record_persistent_model_failure(agent_name, model).await;
556                    }
557                    _ => cooldown::record_credit_exhaustion(agent_name, reason).await,
558                }
559            } else {
560                if let Some(model) = model_name {
561                    response::record_model_failure(agent_name, model).await;
562                }
563                cooldown::record_agent_failure_with_message(agent_name, message).await;
564            }
565            (
566                response::RetryableError::Failed,
567                format!("{agent_name} failed: {message}"),
568            )
569        }
570        agents::AgentError::NetworkError { message } => {
571            // Transient connectivity failure — retry same agent without rerouting,
572            // but grow a dedicated streak counter so backoff keeps increasing.
573            // Use "routed" so it re-dispatches without a full re-routing cycle.
574            // After MAX_NETWORK_RETRIES consecutive network errors, escalate to
575            // needs_review so a human is notified rather than looping forever.
576            const MAX_NETWORK_RETRIES: u64 = 8;
577            let msg = format!("{agent_name} network error: {message}");
578            let retry_count = match store::store_increment(store, repo, task_id, "network_retries")
579                .await
580            {
581                Ok(count) => {
582                    tracing::info!(
583                        task_id,
584                        retry_count = count,
585                        agent = agent_name,
586                        "network retry scheduled"
587                    );
588                    count
589                }
590                Err(e) => {
591                    tracing::warn!(
592                        task_id,
593                        error = %e,
594                        "failed to increment network_retries — escalating to needs_review"
595                    );
596                    return Ok(ErrorHandleResult::Continue {
597                        status: "needs_review".to_string(),
598                        error: format!("{agent_name} network error (store unavailable): {message}"),
599                    });
600                }
601            };
602            if let Err(e) = store::store_set_result(
603                store,
604                repo,
605                task_id,
606                &[("last_error", serde_json::json!(msg))],
607            )
608            .await
609            {
610                tracing::warn!(task_id, err = %e, "failed to write network_error last_error to store");
611            }
612            if retry_count >= MAX_NETWORK_RETRIES {
613                tracing::warn!(
614                    task_id,
615                    retry_count,
616                    agent = agent_name,
617                    "network retries exhausted, escalating to needs_review"
618                );
619                return Ok(ErrorHandleResult::Continue {
620                    status: "needs_review".to_string(),
621                    error: msg,
622                });
623            }
624            // Apply backoff to pace retries
625            response::wait_for_fallback_backoff(task_id, store, repo).await;
626            return Ok(ErrorHandleResult::EarlyReturn {
627                status: "routed".to_string(),
628            });
629        }
630        agents::AgentError::StaleSession { session_id } => {
631            // Stale session errors only occur in the orch chat control session, not
632            // in task runners (tasks don't use --session-id / --resume). Treat as a
633            // generic agent failure so the task is retried from scratch.
634            (
635                response::RetryableError::Failed,
636                format!("{agent_name} stale session: {session_id}"),
637            )
638        }
639        agents::AgentError::ThinkingBlockConflict { message } => {
640            // Transient Anthropic API 400 — thinking blocks modified in multi-turn
641            // tool-use. Not a model capacity or rate-limit problem.
642            // Do NOT record failure counts or model/agent cooldowns.
643            // Pass ModelUnavailable so handle_failover skips agent failure recording
644            // while still updating the reroute chain to avoid re-selecting claude.
645            (
646                response::RetryableError::ModelUnavailable,
647                format!("{agent_name} thinking block conflict: {message}"),
648            )
649        }
650        agents::AgentError::Unknown { exit_code, message } => {
651            // Exit 0 with empty output is a silent failure (common with GitHub
652            // Copilot models in opencode).  Record a model-specific cooldown so
653            // the same model is not retried on every subsequent task.
654            // Also matches when message == "empty-output-exit0" (from classify_from_text
655            // or the direct empty-output branch in mod.rs).
656            if *exit_code == 0
657                && (message.is_empty()
658                    || message == "empty-output-exit0"
659                    || message.starts_with("empty-output-exit0:"))
660            {
661                if let Some(m) = model_name {
662                    // A stopped session also exits 0 with no output, so the backoff stays short
663                    cooldown::record_transient_model_failure(agent_name, m).await;
664                }
665                // Before falling through to handle_failover() (which tries claude/codex),
666                // check whether this agent has any free models that haven't been tried yet.
667                // Silent exit-0 is usually model/provider-specific; retrying other free
668                // models first avoids wasting paid claude/codex failovers.
669                let current = model_name.unwrap_or("");
670                if let Some(result) = try_free_model_reroute(
671                    task_id,
672                    agent_name,
673                    agent_runner,
674                    &[current],
675                    true,
676                    Some("opencode"),
677                    true,
678                    "silent exit 0, retrying with free model",
679                    store,
680                    repo,
681                )
682                .await
683                {
684                    return Ok(result);
685                }
686            }
687
688            // Handle JSON cost telemetry in stdout for non-zero exits
689            // If the message looks like Claude's cost JSON, replace with a generic message
690            let error_message = if *exit_code != 0 && message.contains("\"costUSD\":") {
691                // Extract just the essential info: exit code and that cost telemetry was present
692                format!(
693                    "{agent_name} exited with non-zero status (cost telemetry in stdout, no error message)"
694                )
695            } else {
696                format!("{agent_name} exit {exit_code}: {message}")
697            };
698
699            (response::RetryableError::Failed, error_message)
700        }
701    };
702
703    // Timeouts and interrupts clear the agent and skip the rate_limits row
704    if matches!(
705        retryable,
706        response::RetryableError::Timeout | response::RetryableError::Interrupted
707    ) && let Err(e) = store::store_set_result(
708        store,
709        repo,
710        task_id,
711        &[
712            ("agent", serde_json::json!("")),
713            ("last_error", serde_json::json!(error_msg.clone())),
714        ],
715    )
716    .await
717    {
718        tracing::warn!(task_id, err = %e, "failed to write timeout agent clear to store");
719    }
720
721    // Record rate limit in store (sqlx). Timeouts and interrupted runs are
722    // not usage limits, so they must not write a row
723    {
724        let error_type_str = match retryable {
725            response::RetryableError::UsageLimit => Some("rate"),
726            response::RetryableError::AuthError => Some("auth"),
727            response::RetryableError::Timeout | response::RetryableError::Interrupted => None,
728            _ => Some("error"),
729        };
730        if let Some(limit_type) = error_type_str
731            && let Some(s) = store
732            && let Err(e) = s
733                .record_rate_limit(agent_name, limit_type, Some(task_id))
734                .await
735        {
736            tracing::warn!(task_id, agent = %agent_name, err = %e, "failed to persist rate_limit event — router health check may miss this signal");
737        }
738    }
739
740    // Try free models as last resort before giving up
741    let chain = response::get_reroute_chain(task_id, store, repo).await;
742    let available: Vec<String> = engine::configured_agents()
743        .into_iter()
744        .filter(|a| cmd_cache::command_exists(a))
745        .collect();
746
747    let all_agents_tried = {
748        let chain_set: HashSet<&str> = if chain.is_empty() {
749            HashSet::new()
750        } else {
751            chain.split(',').collect()
752        };
753        !available
754            .iter()
755            .any(|a| a.as_str() != agent_name && !chain_set.contains(a.as_str()))
756    };
757
758    let is_simple = matches!(complexity, None | Some("simple"));
759    if all_agents_tried && is_simple {
760        // All agents exhausted — try free models via opencode (only for simple tasks)
761        if let Some(result) = try_free_model_reroute(
762            task_id,
763            agent_name,
764            agent_runner,
765            &[],
766            true, // fix: check cooldowns (was previously missing, inconsistent with other paths)
767            Some("opencode"),
768            false,
769            "all agents exhausted, trying free model",
770            store,
771            repo,
772        )
773        .await
774        {
775            return Ok(result);
776        }
777    }
778
779    let outcome = response::handle_failover(
780        task_id,
781        agent_name,
782        retryable,
783        &error_msg,
784        credit_recorded,
785        store,
786        repo,
787    )
788    .await;
789    match (outcome.status.as_str(), outcome.next_agent.as_deref()) {
790        ("new", Some(next)) => {
791            tracing::info!(task_id, to = next, "failover: re-routing to fallback agent");
792        }
793        ("new", None) => {
794            tracing::warn!(task_id, "failover exhausted (retryable), task reset to new");
795        }
796        ("blocked", _) => {
797            tracing::warn!(task_id, "failover exhausted (hard failure), task blocked");
798        }
799        _ => {}
800    }
801    let status = outcome.status;
802
803    // Store failure memory for retry learning
804    response::store_failure_memory(
805        task_id,
806        new_attempts,
807        agent_name,
808        model_name,
809        &error_msg,
810        store,
811        repo,
812    )
813    .await;
814
815    Ok(ErrorHandleResult::Continue {
816        status,
817        error: error_msg,
818    })
819}
820
821#[cfg(test)]
822mod tests {
823    use std::time::Duration;
824
825    use super::*;
826    use crate::engine::runner::agents::StreamEvent;
827    use crate::engine::runner::agents::{AgentError, AgentRunner, ParsedResponse, PermissionRules};
828    use crate::parser::AgentResponse;
829    use chrono::Datelike;
830    use serial_test::serial;
831    use tokio::process::Command;
832
833    struct MockRunner {
834        free: Vec<String>,
835    }
836
837    struct MockRunnerWithModels {
838        models: Vec<String>,
839    }
840
841    impl AgentRunner for MockRunnerWithModels {
842        fn stream_events(&self, _event: &serde_json::Value) -> Vec<StreamEvent> {
843            vec![]
844        }
845
846        fn name(&self) -> &'static str {
847            "opencode"
848        }
849
850        fn build_command(
851            &self,
852            _model: Option<&str>,
853            _timeout_cmd: &str,
854            _sys_file: &str,
855            _msg_file: &str,
856            _permissions: &PermissionRules,
857        ) -> String {
858            String::new()
859        }
860
861        fn parse_response(&self, _raw: &str) -> Result<ParsedResponse, AgentError> {
862            Ok(ParsedResponse {
863                response: AgentResponse {
864                    status: "done".to_string(),
865                    summary: String::new(),
866                    accomplished: vec![],
867                    remaining: vec![],
868                    files: vec![],
869                    error: None,
870                    input_tokens: None,
871                    output_tokens: None,
872                    learnings: vec![],
873                    delegations: vec![],
874                    screenshots: vec![],
875                    screenshot_note: None,
876                },
877                input_tokens: None,
878                output_tokens: None,
879                cost_usd: None,
880                duration_ms: None,
881            })
882        }
883
884        fn classify_error(&self, _exit_code: i32, _stdout: &str, _stderr: &str) -> AgentError {
885            AgentError::Unknown {
886                exit_code: 0,
887                message: String::new(),
888            }
889        }
890
891        fn available_models(&self) -> Vec<String> {
892            self.models.clone()
893        }
894
895        fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
896            anyhow::bail!("not implemented")
897        }
898    }
899
900    impl AgentRunner for MockRunner {
901        fn stream_events(&self, _event: &serde_json::Value) -> Vec<StreamEvent> {
902            vec![]
903        }
904
905        fn name(&self) -> &'static str {
906            "opencode"
907        }
908
909        fn build_command(
910            &self,
911            _model: Option<&str>,
912            _timeout_cmd: &str,
913            _sys_file: &str,
914            _msg_file: &str,
915            _permissions: &PermissionRules,
916        ) -> String {
917            String::new()
918        }
919
920        fn parse_response(&self, _raw: &str) -> Result<ParsedResponse, AgentError> {
921            Ok(ParsedResponse {
922                response: AgentResponse {
923                    status: "done".to_string(),
924                    summary: String::new(),
925                    accomplished: vec![],
926                    remaining: vec![],
927                    files: vec![],
928                    error: None,
929                    input_tokens: None,
930                    output_tokens: None,
931                    learnings: vec![],
932                    delegations: vec![],
933                    screenshots: vec![],
934                    screenshot_note: None,
935                },
936                input_tokens: None,
937                output_tokens: None,
938                cost_usd: None,
939                duration_ms: None,
940            })
941        }
942
943        fn classify_error(&self, _exit_code: i32, _stdout: &str, _stderr: &str) -> AgentError {
944            AgentError::Unknown {
945                exit_code: 0,
946                message: String::new(),
947            }
948        }
949
950        fn free_models(&self) -> Vec<String> {
951            self.free.clone()
952        }
953
954        fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
955            anyhow::bail!("not implemented")
956        }
957    }
958
959    /// Verify that a generic (non-credit-exhaustion) rate limit error sets the agent
960    /// cooldown immediately in `handle_error()`, before `handle_failover()` is called.
961    ///
962    /// This prevents concurrent dispatches from starting new runs against the same
963    /// rate-limited agent while the first task is still in its error-handling path.
964    #[serial(cooldown_state)]
965    #[tokio::test]
966    async fn rate_limit_sets_agent_cooldown_immediately() {
967        cooldown::reset_global_state().await;
968        let runner = MockRunner { free: vec![] };
969        let agent = "test-agent-1371-early-cooldown";
970
971        // Confirm no cooldown before the error.
972        assert!(
973            !cooldown::is_agent_in_cooldown(agent),
974            "agent should not be in cooldown before handle_error"
975        );
976
977        let err = AgentError::RateLimit {
978            message: "429 Too Many Requests".to_string(),
979        };
980
981        // Run handle_error — no store so handle_failover will find no agents and return Continue.
982        let _result = handle_error(
983            "test-1371-a",
984            &err,
985            agent,
986            &runner,
987            Some("sonnet"),
988            Some("medium"),
989            1,
990            &None,
991            "owner/repo",
992        )
993        .await
994        .unwrap();
995
996        // Cooldown must be set immediately after handle_error returns, not deferred.
997        assert!(
998            cooldown::is_agent_in_cooldown(agent),
999            "agent should be in cooldown immediately after handle_error for RateLimit"
1000        );
1001    }
1002
1003    /// Verify that a rate limit error sets BOTH the agent-level and model-level cooldowns.
1004    ///
1005    /// This is critical: the router's model selection (model_for_complexity) checks
1006    /// is_model_in_cooldown(agent, model) which uses the agent:model key format.
1007    /// An agent-level cooldown alone does NOT prevent the same model from being
1008    /// re-selected, causing the rate-limited model to fail repeatedly (issue #2153).
1009    #[serial(cooldown_state)]
1010    #[tokio::test]
1011    async fn rate_limit_sets_both_agent_and_model_cooldowns() {
1012        cooldown::reset_global_state().await;
1013        let runner = MockRunner { free: vec![] };
1014        let agent = "test-agent-2153-model-cooldown";
1015        let model = "opencode/github-copilot/qwen3.6-plus-free";
1016
1017        // Confirm no cooldowns before the error.
1018        assert!(
1019            !cooldown::is_agent_in_cooldown(agent),
1020            "agent should not be in cooldown before handle_error"
1021        );
1022        assert!(
1023            !cooldown::is_model_in_cooldown(agent, model),
1024            "model should not be in cooldown before handle_error"
1025        );
1026
1027        let err = AgentError::RateLimit {
1028            message: "Upstream error from Alibaba: Request rate increased too quickly.".to_string(),
1029        };
1030
1031        let _result = handle_error(
1032            "test-2153-a",
1033            &err,
1034            agent,
1035            &runner,
1036            Some(model),
1037            Some("simple"),
1038            1,
1039            &None,
1040            "owner/repo",
1041        )
1042        .await
1043        .unwrap();
1044
1045        // Agent-level cooldown must be set immediately.
1046        assert!(
1047            cooldown::is_agent_in_cooldown(agent),
1048            "agent should be in agent-level cooldown after RateLimit"
1049        );
1050        // Model-level cooldown must ALSO be set immediately — this is the fix for issue #2153.
1051        // Without this, model_for_complexity would still pick the same model.
1052        assert!(
1053            cooldown::is_model_in_cooldown(agent, model),
1054            "model should be in model-level cooldown after RateLimit (fixes issue #2153)"
1055        );
1056    }
1057
1058    /// Verify that rate limit for a specific model does NOT cool other models of the same agent.
1059    #[serial(cooldown_state)]
1060    #[tokio::test]
1061    async fn rate_limit_cooldown_is_model_specific() {
1062        cooldown::reset_global_state().await;
1063        let runner = MockRunner { free: vec![] };
1064        let agent = "test-agent-2153-model-specific";
1065        let cooled_model = "opencode/github-copilot/qwen3.6-plus-free";
1066        let other_model = "opencode/minimax-m2.5-free";
1067
1068        let err = AgentError::RateLimit {
1069            message: "rate limit exceeded".to_string(),
1070        };
1071
1072        let _result = handle_error(
1073            "test-2153-b",
1074            &err,
1075            agent,
1076            &runner,
1077            Some(cooled_model),
1078            Some("simple"),
1079            1,
1080            &None,
1081            "owner/repo",
1082        )
1083        .await
1084        .unwrap();
1085
1086        // The specific model that hit the rate limit should be cooled.
1087        assert!(
1088            cooldown::is_model_in_cooldown(agent, cooled_model),
1089            "rate-limited model should be in cooldown"
1090        );
1091        // Other models of the same agent should NOT be cooled.
1092        assert!(
1093            !cooldown::is_model_in_cooldown(agent, other_model),
1094            "other models should not be cooled by a single model's rate limit"
1095        );
1096    }
1097
1098    /**
1099     * Regression test for issue #3652: a credit-exhaustion error records its
1100     * own credit_failure_count cooldown, but handle_failover() used to also
1101     * bump the generic failure_count:<agent> counter. After a few credit
1102     * errors the next ordinary failure would start at the 24h/48h extended
1103     * backoff tier instead of the 5 minute base. The generic counter must be
1104     * untouched here while credit_failure_count is incremented. The agent
1105     * name is deliberately not a configured agent, so handle_failover() finds
1106     * it as a fallback and takes the recording path this test exercises
1107     **/
1108    #[serial(cooldown_state)]
1109    #[tokio::test]
1110    async fn credit_exhaustion_does_not_bump_generic_failure_count() {
1111        cooldown::reset_global_state().await;
1112        let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1113        cooldown::init_cooldown_store(Arc::clone(&store)).await;
1114
1115        let runner = MockRunner { free: vec![] };
1116        let agent = "test-agent-3652-credit-no-bump";
1117
1118        let err = AgentError::RateLimit {
1119            message: "out of credits, please recharge your balance".to_string(),
1120        };
1121
1122        let _result = handle_error(
1123            "test-3652-a",
1124            &err,
1125            agent,
1126            &runner,
1127            None,
1128            Some("simple"),
1129            1,
1130            &Some(Arc::clone(&store)),
1131            "owner/repo",
1132        )
1133        .await
1134        .unwrap();
1135
1136        let credit_count = store
1137            .kv_get(&format!("credit_failure_count:{agent}"))
1138            .await
1139            .unwrap();
1140        assert!(
1141            credit_count.is_some(),
1142            "credit_failure_count should be incremented by the credit error"
1143        );
1144        let generic_count = store
1145            .kv_get(&format!("failure_count:{agent}"))
1146            .await
1147            .unwrap();
1148        assert!(
1149            generic_count.is_none(),
1150            "failure_count must not be bumped by credit errors, got {generic_count:?}"
1151        );
1152    }
1153
1154    /// A timeout is a long task, not an unavailable model. Two timeouts in a
1155    /// row must leave the model routable and write no rate_limits row
1156    #[serial(cooldown_state)]
1157    #[tokio::test]
1158    async fn two_timeouts_leave_model_routable() {
1159        cooldown::reset_global_state().await;
1160        let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1161        cooldown::init_cooldown_store(Arc::clone(&store)).await;
1162
1163        let runner = MockRunner { free: vec![] };
1164        let agent = "opencode";
1165        let model = "opencode/nemotron-3-ultra-free";
1166
1167        let err = AgentError::Timeout {
1168            elapsed: Duration::from_mins(30),
1169        };
1170
1171        for i in 1..=2 {
1172            let _result = handle_error(
1173                &format!("test-3876-timeout-{i}"),
1174                &err,
1175                agent,
1176                &runner,
1177                Some(model),
1178                Some("simple"),
1179                i,
1180                &Some(Arc::clone(&store)),
1181                "owner/repo",
1182            )
1183            .await
1184            .unwrap();
1185        }
1186
1187        assert!(
1188            !cooldown::is_model_in_cooldown(agent, model),
1189            "timeouts must not cool the model"
1190        );
1191        assert!(
1192            !cooldown::is_agent_in_cooldown(agent),
1193            "timeouts must not cool the agent"
1194        );
1195        let counts = store.recent_rate_limit_counts(6).await.unwrap();
1196        assert_eq!(
1197            counts.get(agent).copied().unwrap_or(0),
1198            0,
1199            "timeouts must not write a rate_limits row"
1200        );
1201    }
1202
1203    /// A run that exits without a result envelope retries without cooling the
1204    /// model or writing a rate_limits row
1205    #[serial(cooldown_state)]
1206    #[tokio::test]
1207    async fn interrupted_run_does_not_write_rate_limit() {
1208        cooldown::reset_global_state().await;
1209        let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1210        cooldown::init_cooldown_store(Arc::clone(&store)).await;
1211
1212        let runner = MockRunner { free: vec![] };
1213        let agent = "claude";
1214        let model = "sonnet";
1215        let err = AgentError::Interrupted {
1216            message: "exited without a result envelope".to_string(),
1217        };
1218
1219        let _result = handle_error(
1220            "test-3876-interrupted",
1221            &err,
1222            agent,
1223            &runner,
1224            Some(model),
1225            Some("medium"),
1226            1,
1227            &Some(Arc::clone(&store)),
1228            "owner/repo",
1229        )
1230        .await
1231        .unwrap();
1232
1233        assert!(
1234            !cooldown::is_model_in_cooldown(agent, model),
1235            "interrupted runs must not cool the model"
1236        );
1237        let counts = store.recent_rate_limit_counts(6).await.unwrap();
1238        assert_eq!(
1239            counts.get(agent).copied().unwrap_or(0),
1240            0,
1241            "interrupted runs must not write a rate_limits row"
1242        );
1243    }
1244
1245    /// Regression test for issue #3580: the in-task failover path (this
1246    /// function) previously never called `parse_relative_usage_window()`, so a
1247    /// vendor-advertised window like "weekly (7-day) usage limit" fell through
1248    /// to the generic exponential backoff (5min base) instead of the ~7-day
1249    /// window the vendor advertised. Only the `needs_review` call site in
1250    /// `runner/mod.rs` applied the window fix from #3572/#3574 — this path did
1251    /// not. Verify the model-specific cooldown now matches the parsed window
1252    /// rather than the short exponential-backoff base.
1253    ///
1254    /// Note: `handle_failover()` still applies its own short agent-level
1255    /// cooldown when switching to a fallback agent — that bookkeeping is
1256    /// unrelated to and unaffected by this fix, so it isn't asserted here.
1257    #[serial(cooldown_state)]
1258    #[tokio::test]
1259    async fn rate_limit_with_relative_window_sets_window_length_model_cooldown() {
1260        cooldown::reset_global_state().await;
1261        let runner = MockRunner { free: vec![] };
1262        let agent = "test-agent-3580-kimi";
1263        let model = "opus";
1264        let key = format!("{agent}:{model}");
1265
1266        let err = AgentError::RateLimit {
1267            message: "You've reached your weekly (7-day) usage limit. Your quota will reset when the current 7-day window expires.".to_string(),
1268        };
1269
1270        let _result = handle_error(
1271            "test-3580-a",
1272            &err,
1273            agent,
1274            &runner,
1275            Some(model),
1276            Some("medium"),
1277            1,
1278            &None,
1279            "owner/repo",
1280        )
1281        .await
1282        .unwrap();
1283
1284        let now = chrono::Utc::now().timestamp();
1285        let until = cooldown::cooldown_until(&key)
1286            .expect("relative usage window should set model cooldown");
1287        let remaining = until.saturating_sub(now);
1288
1289        assert!(
1290            remaining >= 7 * 86400 - 30,
1291            "expected ~7-day model cooldown from the parsed window, got {remaining}s"
1292        );
1293        assert_eq!(
1294            cooldown::cooldown_reason(&key).as_deref(),
1295            Some("rate_limit_usage_window"),
1296            "relative usage-window cooldown must be labeled as a rate limit, not silence"
1297        );
1298    }
1299
1300    #[serial(cooldown_state)]
1301    #[tokio::test]
1302    async fn provider_returned_error_sets_short_model_cooldown() {
1303        cooldown::reset_global_state().await;
1304        let runner = MockRunner { free: vec![] };
1305        let agent = "opencode";
1306        let model = "opencode/nemotron-3-super-free";
1307        let key = format!("{agent}:{model}");
1308
1309        let err = AgentError::RateLimit {
1310            message: "Provider returned error".to_string(),
1311        };
1312
1313        let _result = handle_error(
1314            "test-2478-provider-cooldown",
1315            &err,
1316            agent,
1317            &runner,
1318            Some(model),
1319            Some("simple"),
1320            1,
1321            &None,
1322            "owner/repo",
1323        )
1324        .await
1325        .unwrap();
1326
1327        let now = chrono::Utc::now().timestamp();
1328        let until =
1329            cooldown::cooldown_until(&key).expect("provider error should set model cooldown");
1330        let remaining = until.saturating_sub(now);
1331
1332        assert!(
1333            remaining <= cooldown::TRANSIENT_MODEL_BACKOFF_BASE_SECS,
1334            "expected a 5 minute model cooldown, got {remaining}s"
1335        );
1336    }
1337
1338    #[serial(cooldown_state)]
1339    #[tokio::test]
1340    async fn model_unavailable_sets_persistent_model_cooldown_for_codex_style() {
1341        cooldown::reset_global_state().await;
1342        let runner = MockRunner { free: vec![] };
1343        let agent = "codex";
1344        let model = "gpt-5.3-codex";
1345
1346        // Simulate Codex provider JSON error: message contains "model is not supported"
1347        let err = AgentError::ModelUnavailable {
1348            model: model.to_string(),
1349            message: "{\"detail\":\"The 'gpt-5.3-codex' model is not supported\"}".to_string(),
1350        };
1351
1352        let _result = handle_error(
1353            "test-codex-model-unsupported",
1354            &err,
1355            agent,
1356            &runner,
1357            Some(model),
1358            Some("simple"),
1359            1,
1360            &None,
1361            "owner/repo",
1362        )
1363        .await
1364        .unwrap();
1365
1366        // The persistent backoff base is ~4h; ensure cooldown exists and is at least that long minus a margin
1367        let key = format!("{agent}:{model}");
1368        let now = chrono::Utc::now().timestamp();
1369        let until = cooldown::cooldown_until(&key).expect("model cooldown should be set");
1370        let remaining = until.saturating_sub(now);
1371
1372        assert!(
1373            remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
1374            "expected persistent model cooldown (~4h) for codex model-unavailable, got {remaining}s"
1375        );
1376    }
1377
1378    async fn vendor_reset_remaining(agent: &str, model: &str, message: &str) -> i64 {
1379        cooldown::reset_global_state().await;
1380        let runner = MockRunner { free: vec![] };
1381        let err = AgentError::RateLimit {
1382            message: message.to_string(),
1383        };
1384        let _ = handle_error(
1385            "test-3649",
1386            &err,
1387            agent,
1388            &runner,
1389            Some(model),
1390            Some("simple"),
1391            1,
1392            &None,
1393            "owner/repo",
1394        )
1395        .await
1396        .unwrap();
1397        let until = cooldown::cooldown_until(&format!("{agent}:{model}"))
1398            .expect("model cooldown should be set");
1399        until - chrono::Utc::now().timestamp()
1400    }
1401
1402    #[serial(cooldown_state)]
1403    #[tokio::test]
1404    async fn vendor_reset_date_overrides_weekly_window() {
1405        let t = chrono::Local::now() + chrono::Duration::days(2);
1406        let msg = format!(
1407            "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
1408            t.format("%b"),
1409            t.day()
1410        );
1411        let remaining = vendor_reset_remaining("claude", "sonnet", &msg).await;
1412        assert!(
1413            remaining > 3600 && remaining < 3 * 86400,
1414            "expected ~2d cooldown, got {remaining}s"
1415        );
1416    }
1417
1418    #[serial(cooldown_state)]
1419    #[tokio::test]
1420    async fn vendor_reset_date_overrides_billing_cycle_backoff() {
1421        let t = chrono::Local::now() + chrono::Duration::days(3);
1422        let msg = format!(
1423            "You've hit your usage limit. Upgrade to Pro (https://x), or try again at {}.",
1424            t.format("%b %d, %Y %I:%M %p")
1425        );
1426        let remaining = vendor_reset_remaining("codex", "gpt-5.4", &msg).await;
1427        assert!(
1428            remaining > 2 * 86400 + 80000 - 3600 && remaining < 4 * 86400,
1429            "expected ~3d cooldown, got {remaining}s"
1430        );
1431    }
1432
1433    #[serial(cooldown_state)]
1434    #[tokio::test]
1435    async fn agent_failed_insufficient_balance_sets_credit_cooldown() {
1436        cooldown::reset_global_state().await;
1437        let runner = MockRunner { free: vec![] };
1438        let agent = "test-agent-3642-insufficient-balance";
1439
1440        // Confirm no cooldown before the error.
1441        assert!(
1442            !cooldown::is_agent_in_cooldown(agent),
1443            "agent should not be in cooldown before handle_error"
1444        );
1445
1446        let err = agents::AgentError::AgentFailed {
1447            message: "402 Payment Required: insufficient balance".to_string(),
1448        };
1449
1450        // Run handle_error — no store so handle_failover will find no agents and return Continue.
1451        let _result = handle_error(
1452            "test-3642-a",
1453            &err,
1454            agent,
1455            &runner,
1456            Some("sonnet"),
1457            Some("medium"),
1458            1,
1459            &None,
1460            "owner/repo",
1461        )
1462        .await
1463        .unwrap();
1464
1465        // The credit exhaustion path (from #3642) should set a cooldown of at least
1466        // CREDIT_BACKOFF_BASE_SECS (1h), not the generic 5-min backoff.
1467        // Without #3642, the AgentFailed arm would have gone to the else branch
1468        // and called record_agent_failure_with_message, setting a 5-min cooldown.
1469        assert!(
1470            cooldown::is_agent_in_cooldown(agent),
1471            "agent should be in cooldown after handle_error for AgentFailed with insufficient balance"
1472        );
1473
1474        let until = cooldown::cooldown_until(agent).expect("agent cooldown should be active");
1475        let remaining = until - chrono::Utc::now().timestamp();
1476        assert!(
1477            remaining >= cooldown::CREDIT_BACKOFF_BASE_SECS - 30,
1478            "credit cooldown should be at least the 1h base, got {remaining}s"
1479        );
1480    }
1481
1482    #[serial(cooldown_state)]
1483    #[tokio::test]
1484    async fn silent_exit0_retries_free_model_for_simple_complexity() {
1485        cooldown::reset_global_state().await;
1486        let runner = MockRunner {
1487            free: vec!["opencode/mimo-v2-omni-free".to_string()],
1488        };
1489        let err = AgentError::Unknown {
1490            exit_code: 0,
1491            message: String::new(),
1492        };
1493
1494        // complexity=simple → free model retry is appropriate
1495        let result = handle_error(
1496            "test-1112-a",
1497            &err,
1498            "opencode",
1499            &runner,
1500            Some("opencode/gpt-5.4-mini"),
1501            Some("simple"),
1502            1,
1503            &None,
1504            "owner/repo",
1505        )
1506        .await
1507        .unwrap();
1508
1509        assert!(
1510            matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1511            "expected EarlyReturn{{status: new}} for simple complexity — free model should be tried first"
1512        );
1513    }
1514
1515    #[serial(cooldown_state)]
1516    #[tokio::test]
1517    async fn silent_exit0_retries_free_model_when_complexity_unknown() {
1518        cooldown::reset_global_state().await;
1519        let runner = MockRunner {
1520            free: vec!["opencode/mimo-v2-omni-free".to_string()],
1521        };
1522        let err = AgentError::Unknown {
1523            exit_code: 0,
1524            message: String::new(),
1525        };
1526
1527        // complexity=None (unknown) → treated as simple → free model retry
1528        let result = handle_error(
1529            "test-1112-a2",
1530            &err,
1531            "opencode",
1532            &runner,
1533            Some("opencode/gpt-5.4-mini"),
1534            None,
1535            1,
1536            &None,
1537            "owner/repo",
1538        )
1539        .await
1540        .unwrap();
1541
1542        assert!(
1543            matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1544            "expected EarlyReturn{{status: new}} for unknown complexity — free model should be tried first"
1545        );
1546    }
1547
1548    #[serial(cooldown_state)]
1549    #[tokio::test]
1550    async fn silent_exit0_retries_free_model_for_medium_complexity() {
1551        cooldown::reset_global_state().await;
1552        let runner = MockRunner {
1553            free: vec!["opencode/mimo-v2-omni-free".to_string()],
1554        };
1555        let err = AgentError::Unknown {
1556            exit_code: 0,
1557            message: String::new(),
1558        };
1559
1560        // complexity=medium → still retry free model before cross-agent failover
1561        let result = handle_error(
1562            "test-1195-medium",
1563            &err,
1564            "opencode",
1565            &runner,
1566            Some("opencode/copilot-gpt-5.4-mini"),
1567            Some("medium"),
1568            1,
1569            &None,
1570            "owner/repo",
1571        )
1572        .await
1573        .unwrap();
1574
1575        assert!(
1576            matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1577            "expected EarlyReturn{{status: new}} for medium complexity — free model should be tried before failover"
1578        );
1579    }
1580
1581    #[serial(cooldown_state)]
1582    #[tokio::test]
1583    async fn silent_exit0_retries_free_model_for_complex_complexity() {
1584        cooldown::reset_global_state().await;
1585        let runner = MockRunner {
1586            free: vec!["opencode/mimo-v2-omni-free".to_string()],
1587        };
1588        let err = AgentError::Unknown {
1589            exit_code: 0,
1590            message: String::new(),
1591        };
1592
1593        // complexity=complex → still retry free model before cross-agent failover
1594        let result = handle_error(
1595            "test-1195-complex",
1596            &err,
1597            "opencode",
1598            &runner,
1599            Some("opencode/copilot-gpt-5.4-mini"),
1600            Some("complex"),
1601            1,
1602            &None,
1603            "owner/repo",
1604        )
1605        .await
1606        .unwrap();
1607
1608        assert!(
1609            matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1610            "expected EarlyReturn{{status: new}} for complex complexity — free model should be tried before failover"
1611        );
1612    }
1613
1614    /// Context overflow is deterministic — the same context will overflow every agent.
1615    /// Verify that it escalates directly to needs_review without cycling through fallback agents.
1616    #[serial(cooldown_state)]
1617    #[tokio::test]
1618    async fn context_overflow_escalates_to_needs_review_immediately() {
1619        cooldown::reset_global_state().await;
1620        let runner = MockRunner { free: vec![] };
1621        let err = AgentError::ContextOverflow {
1622            message: "prompt is too long: 250000 tokens, max 200000".to_string(),
1623        };
1624
1625        let result = handle_error(
1626            "test-2129-a",
1627            &err,
1628            "claude",
1629            &runner,
1630            Some("sonnet"),
1631            Some("medium"),
1632            1,
1633            &None,
1634            "owner/repo",
1635        )
1636        .await
1637        .unwrap();
1638
1639        assert!(
1640            matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "needs_review"),
1641            "context overflow must escalate directly to needs_review without cycling through fallback agents"
1642        );
1643    }
1644
1645    /// When store is None, store_set_result returns Ok(()) — reroute proceeds normally.
1646    /// This verifies the happy path is not broken by the error-propagation change.
1647    #[serial(cooldown_state)]
1648    #[tokio::test]
1649    async fn free_model_reroute_succeeds_without_store() {
1650        cooldown::reset_global_state().await;
1651        let runner = MockRunner {
1652            free: vec!["opencode/mimo-v2-omni-free".to_string()],
1653        };
1654        let err = AgentError::Unknown {
1655            exit_code: 0,
1656            message: String::new(),
1657        };
1658
1659        // store=None → store_set_result returns Ok(()) → reroute proceeds
1660        let result = handle_error(
1661            "test-2293-a",
1662            &err,
1663            "opencode",
1664            &runner,
1665            Some("opencode/gpt-5.4-mini"),
1666            Some("simple"),
1667            1,
1668            &None,
1669            "owner/repo",
1670        )
1671        .await
1672        .unwrap();
1673
1674        assert!(
1675            matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1676            "expected EarlyReturn{{status: new}} — store_set_result with None store should not prevent reroute"
1677        );
1678    }
1679
1680    #[serial(cooldown_state)]
1681    #[tokio::test]
1682    async fn silent_exit0_falls_through_when_no_free_models() {
1683        cooldown::reset_global_state().await;
1684        let runner = MockRunner { free: vec![] };
1685        let err = AgentError::Unknown {
1686            exit_code: 0,
1687            message: String::new(),
1688        };
1689
1690        let result = handle_error(
1691            "test-1112-b",
1692            &err,
1693            "opencode",
1694            &runner,
1695            Some("opencode/gpt-5.4-mini"),
1696            Some("simple"),
1697            1,
1698            &None,
1699            "owner/repo",
1700        )
1701        .await
1702        .unwrap();
1703
1704        assert!(
1705            matches!(result, ErrorHandleResult::Continue { .. }),
1706            "expected Continue (fallthrough to failover) when no free models available"
1707        );
1708    }
1709
1710    /// Verify that InvalidResponse applies model-level cooldown.
1711    ///
1712    /// This fixes issue #2750: InvalidResponse used to skip model-level cooldown,
1713    /// allowing models that produced unparseable responses to be immediately
1714    /// retried on the next task without any backoff.
1715    #[serial(cooldown_state)]
1716    #[tokio::test]
1717    async fn invalid_response_sets_model_cooldown() {
1718        cooldown::reset_global_state().await;
1719        let runner = MockRunner { free: vec![] };
1720        let agent = "opencode";
1721        let model = "opencode/nemotron-3-super-free";
1722
1723        // Confirm no model cooldown before the error.
1724        assert!(
1725            !cooldown::is_model_in_cooldown(agent, model),
1726            "model should not be in cooldown before handle_error"
1727        );
1728
1729        let err = AgentError::InvalidResponse {
1730            raw: "invalid json output".to_string(),
1731        };
1732
1733        let _result = handle_error(
1734            "test-2750-invalid-response",
1735            &err,
1736            agent,
1737            &runner,
1738            Some(model),
1739            Some("simple"),
1740            1,
1741            &None,
1742            "owner/repo",
1743        )
1744        .await
1745        .unwrap();
1746
1747        // Model-level cooldown must be set after InvalidResponse.
1748        assert!(
1749            cooldown::is_model_in_cooldown(agent, model),
1750            "model should be in cooldown after InvalidResponse (fixes issue #2750)"
1751        );
1752    }
1753
1754    /// BillingCycleExhausted with a known model must set ONLY the model-level cooldown.
1755    ///
1756    /// Regression test for issue #3382: "upgrade to pro" messages (added to
1757    /// billing_cycle_patterns in #3380) were triggering an agent-wide 24h cooldown
1758    /// when model_name was None, blocking all codex models even though only the
1759    /// specific model's plan quota was exhausted.
1760    #[serial(cooldown_state)]
1761    #[tokio::test]
1762    async fn billing_cycle_exhausted_with_known_model_sets_only_model_cooldown() {
1763        cooldown::reset_global_state().await;
1764        let runner = MockRunner { free: vec![] };
1765        let agent = "codex-3382-billing-cycle";
1766        let model = "gpt-5.5";
1767
1768        let err = AgentError::RateLimit {
1769            message: "rate limit: You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro).".to_string(),
1770        };
1771
1772        let _result = handle_error(
1773            "test-3382-a",
1774            &err,
1775            agent,
1776            &runner,
1777            Some(model),
1778            Some("medium"),
1779            1,
1780            &None,
1781            "owner/repo",
1782        )
1783        .await
1784        .unwrap();
1785
1786        // Agent-level credit cooldown must NOT be set — the quota is model-scoped.
1787        assert!(
1788            !cooldown::is_agent_in_cooldown(agent),
1789            "agent must NOT be in agent-level cooldown when BillingCycleExhausted has a known model (issue #3382)"
1790        );
1791        // Model-level cooldown MUST be set.
1792        assert!(
1793            cooldown::is_model_in_cooldown(agent, model),
1794            "model must be in cooldown after BillingCycleExhausted (issue #3382)"
1795        );
1796        // Other models on the same agent must NOT be cooled.
1797        assert!(
1798            !cooldown::is_model_in_cooldown(agent, "gpt-5.4"),
1799            "other models on the same agent must not be cooled by a single model's billing exhaustion"
1800        );
1801    }
1802
1803    #[serial(cooldown_state)]
1804    #[tokio::test]
1805    async fn summarize_rate_limit_error_extracts_api_retry_info() {
1806        cooldown::reset_global_state().await;
1807        let raw_output = r#"some log lines
1808{"type":"system","subtype":"api_retry","attempt":6,"max_retries":10,"retry_delay_ms":17266.789,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1809{"type":"system","subtype":"api_retry","attempt":7,"max_retries":10,"retry_delay_ms":35162.66,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1810more logs"#;
1811
1812        let summarized = super::summarize_rate_limit_error(raw_output);
1813        assert_eq!(summarized, "status=429 after 7 attempts (last delay 35s)");
1814    }
1815
1816    #[serial(cooldown_state)]
1817    #[tokio::test]
1818    async fn summarize_rate_limit_error_returns_original_when_no_api_retry() {
1819        cooldown::reset_global_state().await;
1820        let raw_output = "some regular error message without api_retry JSON";
1821
1822        let summarized = super::summarize_rate_limit_error(raw_output);
1823        assert_eq!(summarized, raw_output);
1824    }
1825
1826    #[serial(cooldown_state)]
1827    #[tokio::test]
1828    async fn summarize_rate_limit_error_handles_malformed_json_gracefully() {
1829        cooldown::reset_global_state().await;
1830        let raw_output = r#"some logs
1831{"type":"system","subtype":"api_retry","attempt":"not_a_number","retry_delay_ms":invalid,"error_status":429}
1832more logs"#;
1833
1834        let summarized = super::summarize_rate_limit_error(raw_output);
1835        // Should still return a formatted string even with invalid numbers
1836        assert!(summarized.starts_with("status="));
1837        assert!(summarized.contains("after"));
1838        assert!(summarized.contains("attempts"));
1839        assert!(summarized.contains("(last delay"));
1840    }
1841
1842    #[serial(cooldown_state)]
1843    #[tokio::test]
1844    async fn summarize_rate_limit_error_works_with_single_line() {
1845        cooldown::reset_global_state().await;
1846        let raw_output = r#"{"type":"system","subtype":"api_retry","attempt":3,"max_retries":10,"retry_delay_ms":5000,"error_status":429,"error":"rate_limit"}"#;
1847
1848        let summarized = super::summarize_rate_limit_error(raw_output);
1849        assert_eq!(summarized, "status=429 after 3 attempts (last delay 5s)");
1850    }
1851
1852    /// Verify extract_json_field correctly handles quoted string values.
1853    /// Previously it stripped from the wrong end (strip_suffix), returning None for
1854    /// any field that isn't the last one in the JSON object (fixes issue #2817).
1855    #[serial(cooldown_state)]
1856    #[tokio::test]
1857    async fn extract_json_field_handles_quoted_strings() {
1858        cooldown::reset_global_state().await;
1859        let json = r#"{"error_status":429,"attempt":7,"retry_delay_ms":35162.66,"error":"rate_limit","session_id":"331c099b-..."}"#;
1860
1861        assert_eq!(
1862            super::extract_json_field(json, "\"error\":"),
1863            Some("rate_limit".to_string()),
1864            "quoted string field mid-object should be extracted correctly"
1865        );
1866        assert_eq!(
1867            super::extract_json_field(json, "\"session_id\":"),
1868            Some("331c099b-...".to_string()),
1869            "quoted string field at end of object (no trailing comma) should be extracted"
1870        );
1871        // Numeric fields should still work (regression check)
1872        assert_eq!(
1873            super::extract_json_field(json, "\"error_status\":"),
1874            Some("429".to_string())
1875        );
1876        assert_eq!(
1877            super::extract_json_field(json, "\"attempt\":"),
1878            Some("7".to_string())
1879        );
1880        assert_eq!(
1881            super::extract_json_field(json, "\"retry_delay_ms\":"),
1882            Some("35162.66".to_string())
1883        );
1884    }
1885
1886    /// Truncated fragment starting mid-object with ,"subtype":"api_retry"...
1887    /// Should NOT return raw JSON blob (regression test for issue #2839).
1888    #[serial(cooldown_state)]
1889    #[tokio::test]
1890    async fn summarize_rate_limit_error_handles_truncated_fragment_with_subtype() {
1891        cooldown::reset_global_state().await;
1892        // Real-world truncated fragment from GLM runs (no "type":"system" prefix)
1893        let raw_output = r#"some log context
1894,"subtype":"api_retry","attempt":5,"retry_delay_ms":20000,"error_status":429,"error":"rate_limit"
1895more context"#;
1896
1897        let summarized = super::summarize_rate_limit_error(raw_output);
1898        assert!(
1899            !summarized.contains("subtype"),
1900            "summarized output must not contain raw 'subtype' JSON fragment"
1901        );
1902        assert!(
1903            !summarized.contains('{'),
1904            "summarized output must not contain JSON braces"
1905        );
1906        assert!(
1907            summarized.starts_with("status="),
1908            "summarized output should start with 'status=' — got: {summarized}"
1909        );
1910        // Note: "api_retry" is only included when fields are not extractable
1911        // (the fragment fallback path). When attempt/delay are extractable,
1912        // the summary is "status=X after N attempts (last delay Ys)" with no "api_retry".
1913    }
1914
1915    /// Fragment with attempt and delay fields extractable from mid-fragment.
1916    #[serial(cooldown_state)]
1917    #[tokio::test]
1918    async fn summarize_rate_limit_error_handles_fragment_with_extractable_fields() {
1919        cooldown::reset_global_state().await;
1920        let raw_output = r#"log output
1921,"subtype":"api_retry","attempt":5,"retry_delay_ms":20000,"error_status":429,"error":"rate_limit","session_id":"abc123","max_retries":10}
1922rest of output"#;
1923
1924        let summarized = super::summarize_rate_limit_error(raw_output);
1925        assert_eq!(summarized, "status=429 after 5 attempts (last delay 20s)");
1926    }
1927
1928    /// Complete JSON (no truncation) still works after removing "type":"system" requirement.
1929    #[serial(cooldown_state)]
1930    #[tokio::test]
1931    async fn summarize_rate_limit_error_complete_json_still_works() {
1932        cooldown::reset_global_state().await;
1933        let raw_output = r#"some log lines
1934{"type":"system","subtype":"api_retry","attempt":6,"max_retries":10,"retry_delay_ms":17266.789,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1935{"type":"system","subtype":"api_retry","attempt":7,"max_retries":10,"retry_delay_ms":35162.66,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1936more logs"#;
1937
1938        let summarized = super::summarize_rate_limit_error(raw_output);
1939        assert_eq!(summarized, "status=429 after 7 attempts (last delay 35s)");
1940    }
1941
1942    /// When no api_retry JSON is found at all, should NOT return raw passthrough.
1943    #[serial(cooldown_state)]
1944    #[tokio::test]
1945    async fn summarize_rate_limit_error_no_api_retry_no_raw_passthrough() {
1946        cooldown::reset_global_state().await;
1947        let raw_output = "some regular error message without api_retry JSON that is quite long and should be truncated";
1948
1949        let summarized = super::summarize_rate_limit_error(raw_output);
1950        // Should be truncated to first line, not the full raw_output
1951        assert!(
1952            summarized.len() <= 140,
1953            "expected truncated first-line, got: {summarized}"
1954        );
1955    }
1956
1957    /// Short error without api_retry: should return the line as-is (no truncation needed).
1958    #[serial(cooldown_state)]
1959    #[tokio::test]
1960    async fn summarize_rate_limit_error_no_api_retry_short() {
1961        cooldown::reset_global_state().await;
1962        let raw_output = "rate limit exceeded";
1963
1964        let summarized = super::summarize_rate_limit_error(raw_output);
1965        assert_eq!(summarized, raw_output);
1966    }
1967
1968    /// "Model not found" is a permanent failure — verify it applies the persistent (4h base)
1969    /// cooldown rather than the standard transient (5min base) cooldown.
1970    ///
1971    /// Regression test for issue #2941: dead models were retried every 4h indefinitely
1972    /// because "not found" was treated the same as a transient unavailability.
1973    #[serial(cooldown_state)]
1974    #[tokio::test]
1975    async fn model_not_found_applies_persistent_cooldown() {
1976        cooldown::reset_global_state().await;
1977        let runner = MockRunner { free: vec![] };
1978        let agent = "opencode-2941-not-found";
1979        let model = "github-copilot/claude-opus-4.6";
1980        let key = format!("{agent}:{model}");
1981
1982        let err = AgentError::ModelUnavailable {
1983            model: model.to_string(),
1984            message: "Model not found: github-copilot/claude-opus-4.6".to_string(),
1985        };
1986
1987        let _result = handle_error(
1988            "test-2941-a",
1989            &err,
1990            agent,
1991            &runner,
1992            Some(model),
1993            Some("complex"),
1994            1,
1995            &None,
1996            "owner/repo",
1997        )
1998        .await
1999        .unwrap();
2000
2001        let now = chrono::Utc::now().timestamp();
2002        let until =
2003            cooldown::cooldown_until(&key).expect("model not found should set model cooldown");
2004        let remaining = until.saturating_sub(now);
2005
2006        // Persistent cooldown starts at 4h; transient starts at 5min.
2007        // Verify we applied the persistent (4h base) path, not the short one.
2008        assert!(
2009            remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2010            "expected ~4h (persistent) cooldown for 'Model not found', got {remaining}s — \
2011             may have applied transient 5min cooldown instead"
2012        );
2013    }
2014
2015    /// "Model not supported" is a permanent account-type mismatch — verify it applies the
2016    /// persistent (4h base → 7d max) cooldown, same as "not found".
2017    ///
2018    /// Regression test for issue #3215: "not supported" was not matched by the
2019    /// `is_permanently_gone` check, so it fell through to the 5min→4h transient backoff
2020    /// and the model kept being retried every ~4h indefinitely.
2021    #[serial(cooldown_state)]
2022    #[tokio::test]
2023    async fn model_not_supported_applies_persistent_cooldown() {
2024        cooldown::reset_global_state().await;
2025        let runner = MockRunner { free: vec![] };
2026        let agent = "codex-3215-not-supported";
2027        let model = "gpt-5.2-codex";
2028        let key = format!("{agent}:{model}");
2029
2030        let err = AgentError::ModelUnavailable {
2031            model: model.to_string(),
2032            message: "The 'gpt-5.2-codex' model is not supported when using Codex with a ChatGPT account.".to_string(),
2033        };
2034
2035        let _result = handle_error(
2036            "test-3215-a",
2037            &err,
2038            agent,
2039            &runner,
2040            Some(model),
2041            Some("medium"),
2042            1,
2043            &None,
2044            "owner/repo",
2045        )
2046        .await
2047        .unwrap();
2048
2049        let now = chrono::Utc::now().timestamp();
2050        let until =
2051            cooldown::cooldown_until(&key).expect("model not supported should set model cooldown");
2052        let remaining = until.saturating_sub(now);
2053
2054        assert!(
2055            remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2056            "expected ~4h (persistent) cooldown for 'not supported', got {remaining}s — \
2057             may have applied transient 5min cooldown instead (issue #3215)"
2058        );
2059    }
2060
2061    #[serial(cooldown_state)]
2062    #[tokio::test]
2063    async fn model_not_available_in_country_applies_persistent_cooldown() {
2064        cooldown::reset_global_state().await;
2065        let runner = MockRunner { free: vec![] };
2066        let agent = "opencode-3650-country";
2067        let model = "some-region-locked-model";
2068        let key = format!("{agent}:{model}");
2069
2070        // issue #3650: region/country blocks need the persistent backoff, not the 5min one
2071        for message in [
2072            "This model is not available in your country.",
2073            "This model is not available in your region.",
2074        ] {
2075            cooldown::reset_global_state().await;
2076            let err = AgentError::ModelUnavailable {
2077                model: model.to_string(),
2078                message: message.to_string(),
2079            };
2080
2081            let _result = handle_error(
2082                "test-3650",
2083                &err,
2084                agent,
2085                &runner,
2086                Some(model),
2087                Some("medium"),
2088                1,
2089                &None,
2090                "owner/repo",
2091            )
2092            .await
2093            .unwrap();
2094
2095            let now = chrono::Utc::now().timestamp();
2096            let until =
2097                cooldown::cooldown_until(&key).expect("region block should set model cooldown");
2098            let remaining = until.saturating_sub(now);
2099
2100            assert!(
2101                remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2102                "expected ~4h (persistent) cooldown for '{message}', got {remaining}s — \
2103                 may have applied transient 5min cooldown instead (issue #3650)"
2104            );
2105        }
2106    }
2107
2108    /// ThinkingBlockConflict must NOT set any agent or model cooldown.
2109    ///
2110    /// This error is a transient Claude CLI 400 (thinking blocks modified in multi-turn
2111    /// tool-use) — not a model capacity problem. Penalising claude:opus with failure
2112    /// counts would prematurely push it into extended-backoff for unrelated reasons.
2113    #[serial(cooldown_state)]
2114    #[tokio::test]
2115    async fn thinking_block_conflict_does_not_set_cooldown() {
2116        cooldown::reset_global_state().await;
2117        let runner = MockRunner { free: vec![] };
2118        let agent = "claude-3204-thinking-block";
2119        let model = "opus";
2120
2121        assert!(
2122            !cooldown::is_agent_in_cooldown(agent),
2123            "agent must not be in cooldown before handle_error"
2124        );
2125        assert!(
2126            !cooldown::is_model_in_cooldown(agent, model),
2127            "model must not be in cooldown before handle_error"
2128        );
2129
2130        let err = AgentError::ThinkingBlockConflict {
2131            message: "`thinking` or `redacted_thinking` blocks in the latest assistant message cannot be modified.".to_string(),
2132        };
2133
2134        let _result = handle_error(
2135            "test-3204-thinking",
2136            &err,
2137            agent,
2138            &runner,
2139            Some(model),
2140            Some("complex"),
2141            1,
2142            &None,
2143            "owner/repo",
2144        )
2145        .await
2146        .unwrap();
2147
2148        assert!(
2149            !cooldown::is_agent_in_cooldown(agent),
2150            "agent must NOT be in cooldown after ThinkingBlockConflict (fixes issue #3204)"
2151        );
2152        assert!(
2153            !cooldown::is_model_in_cooldown(agent, model),
2154            "model must NOT be in cooldown after ThinkingBlockConflict (fixes issue #3204)"
2155        );
2156    }
2157
2158    /// Transient model unavailability (not "not found") should use the standard short cooldown.
2159    #[serial(cooldown_state)]
2160    #[tokio::test]
2161    async fn model_unavailable_transient_applies_short_cooldown() {
2162        cooldown::reset_global_state().await;
2163        let runner = MockRunner { free: vec![] };
2164        let agent = "opencode-2941-transient";
2165        let model = "github-copilot/gpt-4o";
2166        let key = format!("{agent}:{model}");
2167
2168        let err = AgentError::ModelUnavailable {
2169            model: model.to_string(),
2170            message: "No endpoints found for github-copilot/gpt-4o.".to_string(),
2171        };
2172
2173        let _result = handle_error(
2174            "test-2941-b",
2175            &err,
2176            agent,
2177            &runner,
2178            Some(model),
2179            Some("medium"),
2180            1,
2181            &None,
2182            "owner/repo",
2183        )
2184        .await
2185        .unwrap();
2186
2187        let now = chrono::Utc::now().timestamp();
2188        let until = cooldown::cooldown_until(&key)
2189            .expect("transient model unavailability should set model cooldown");
2190        let remaining = until.saturating_sub(now);
2191
2192        // Standard (transient) backoff base is 5min. Persistent base is 4h.
2193        // Confirm we did NOT apply the long 4h cooldown for a transient error.
2194        assert!(
2195            remaining < cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2196            "transient unavailability should use short cooldown (<4h), got {remaining}s"
2197        );
2198    }
2199
2200    #[serial(cooldown_state)]
2201    #[tokio::test]
2202    async fn model_unavailable_does_not_reuse_same_agent_when_agent_cooled() {
2203        cooldown::reset_global_state().await;
2204        let agent = "codex-3620";
2205        let failing_model = "gpt-5.4";
2206        let other_model = "gpt-5.5";
2207        let runner = MockRunnerWithModels {
2208            models: vec![failing_model.to_string(), other_model.to_string()],
2209        };
2210
2211        cooldown::set_agent_cooldown(agent, 3600).await;
2212
2213        let err = AgentError::ModelUnavailable {
2214            model: failing_model.to_string(),
2215            message: "model gpt-5.4 unavailable".to_string(),
2216        };
2217
2218        let result = handle_error(
2219            "test-3620",
2220            &err,
2221            agent,
2222            &runner,
2223            Some(failing_model),
2224            Some("medium"),
2225            1,
2226            &None,
2227            "owner/repo",
2228        )
2229        .await
2230        .unwrap();
2231
2232        if let ErrorHandleResult::EarlyReturn { status } = result {
2233            assert_ne!(
2234                status, "routed",
2235                "agent-wide cooldown was defeated by same-agent model substitution"
2236            );
2237        }
2238    }
2239}