1use crate::store;
8use crate::store::TaskStore;
9use std::collections::HashSet;
10use std::sync::Arc;
11
12use super::{agents, response};
13use crate::cmd_cache;
14use crate::engine;
15use crate::engine::cooldown;
16use crate::engine::cooldown::CreditExhaustionReason;
17
18pub(crate) fn summarize_rate_limit_error(raw_output: &str) -> String {
29 const FIRST_LINE_CAP: usize = 120;
31 let last_api_retry_line = raw_output
35 .lines()
36 .rev()
37 .find(|line| line.contains("\"subtype\":\"api_retry\""));
38
39 if let Some(line) = last_api_retry_line {
40 return summarise_api_retry_line(line);
41 }
42
43 let api_retry_field_markers = ["\"error_status\":", "\"attempt\":", "\"retry_delay_ms\":"];
49 let field_line = raw_output
50 .lines()
51 .rev()
52 .find(|line| api_retry_field_markers.iter().any(|m| line.contains(m)));
53 if let Some(line) = field_line {
54 return summarise_api_retry_line(line);
55 }
56
57 let first_line = raw_output.lines().next().unwrap_or(raw_output);
61 if first_line.len() > FIRST_LINE_CAP {
62 let cutoff = agents::truncate_at_char_boundary(first_line, FIRST_LINE_CAP);
63 let remaining = raw_output.len().saturating_sub(cutoff);
64 format!("{}... (+{} more chars)", &first_line[..cutoff], remaining)
65 } else {
66 first_line.to_string()
67 }
68}
69
70fn summarise_api_retry_line(line: &str) -> String {
75 let error_status =
76 extract_json_field(line, "\"error_status\":").unwrap_or_else(|| "unknown".to_string());
77 let attempt = extract_json_field(line, "\"attempt\":").unwrap_or_else(|| "unknown".to_string());
78 let retry_delay_ms =
79 extract_json_field(line, "\"retry_delay_ms\":").unwrap_or_else(|| "0".to_string());
80
81 let has_field = error_status != "unknown"
82 || attempt != "unknown"
83 || (retry_delay_ms != "0" && retry_delay_ms != "unknown");
84
85 if has_field {
86 let retry_delay_s = retry_delay_ms.parse::<f64>().unwrap_or(0.0) / 1000.0;
87 #[expect(
88 clippy::cast_possible_truncation,
89 clippy::cast_sign_loss,
90 reason = "the saturating cast floors the delay to whole seconds"
91 )]
92 let retry_delay_s = retry_delay_s as u64;
93 format!("status={error_status} after {attempt} attempts (last delay {retry_delay_s}s)")
94 } else {
95 let status_hint = extract_json_field(line, "\"status\":")
96 .or_else(|| extract_json_field(line, "\"error\":"))
97 .unwrap_or_else(|| "429".to_string());
98 format!("status={status_hint} (api_retry fragment)")
99 }
100}
101
102fn extract_json_field(json: &str, field: &str) -> Option<String> {
104 json.find(field).and_then(|start| {
105 let value_start = start + field.len();
106 let value_str = &json[value_start..];
107
108 if let Some(rest) = value_str.strip_prefix('"') {
110 let end = rest.find('"')?; Some(rest[..end].to_string())
112 }
113 else {
115 let end = value_str
116 .find(|c: char| !c.is_ascii_digit() && c != '.')
117 .unwrap_or(value_str.len());
118 let num_str = &value_str[..end];
119 num_str
120 .strip_suffix(',')
121 .map(ToString::to_string)
122 .or_else(|| Some(num_str.to_string()))
123 }
124 })
125}
126
127#[derive(Debug)]
129pub enum ErrorHandleResult {
130 EarlyReturn { status: String },
132 Continue { status: String, error: String },
138}
139
140#[expect(
151 clippy::too_many_arguments,
152 reason = "each caller picks a different mix of reroute options"
153)]
154#[expect(
155 clippy::ref_option,
156 reason = "the store helpers it calls take the store as &Option"
157)]
158async fn try_free_model_reroute(
159 task_id: &str,
160 agent_name: &str,
161 agent_runner: &dyn agents::AgentRunner,
162 exclude_models: &[&str],
163 check_cooldowns: bool,
164 set_agent: Option<&str>,
165 apply_backoff: bool,
166 reason_prefix: &str,
167 store: &Option<Arc<TaskStore>>,
168 repo: &str,
169) -> Option<ErrorHandleResult> {
170 let free = agent_runner.free_models();
171 if free.is_empty() {
172 return None;
173 }
174 let tried_models: String = store::opt_store_get_task(store, repo, task_id)
175 .await
176 .map(|t| t.model_reroute_chain)
177 .unwrap_or_default();
178 let tried_set: HashSet<&str> = tried_models.split(',').filter(|s| !s.is_empty()).collect();
179 let next_free = free.iter().find(|m| {
180 !exclude_models.contains(&m.as_str())
181 && !tried_set.contains(m.as_str())
182 && (!check_cooldowns
183 || (!cooldown::is_agent_in_cooldown(agent_name)
184 && !response::is_model_in_cooldown(agent_name, m)))
185 })?;
186 tracing::info!(task_id, model = %next_free, "{}", reason_prefix);
187 let new_tried = if tried_models.is_empty() {
188 next_free.clone()
189 } else {
190 format!("{tried_models},{next_free}")
191 };
192 let msg = format!("{reason_prefix} {next_free}");
193 let mut fields: Vec<(&str, serde_json::Value)> = vec![
194 ("model", serde_json::json!(next_free.clone())),
195 ("model_reroute_chain", serde_json::json!(new_tried)),
196 ("last_error", serde_json::json!(msg)),
197 ];
198 if let Some(agent) = set_agent {
199 fields.push(("agent", serde_json::json!(agent)));
200 }
201 if let Err(e) = store::store_set_result(store, repo, task_id, &fields).await {
202 tracing::warn!(task_id, err = %e, "try_free_model_reroute: failed to persist model update — not rerouting");
203 return None; }
205 if apply_backoff {
206 response::wait_for_fallback_backoff(task_id, store, repo).await;
207 }
208 Some(ErrorHandleResult::EarlyReturn {
209 status: "new".to_string(),
210 })
211}
212
213async fn apply_vendor_reset_date(
218 agent_name: &str,
219 model_name: Option<&str>,
220 message: &str,
221) -> bool {
222 let Some(model) = model_name else {
223 return false;
224 };
225 match cooldown::detect_credit_exhaustion(message) {
226 None | Some(CreditExhaustionReason::BillingCycleExhausted) => {}
227 Some(_) => return false,
228 }
229 match cooldown::parse_retry_at(message) {
230 Some(until) if until > chrono::Utc::now().timestamp() => {
231 cooldown::set_model_cooldown_until(agent_name, model, until).await;
232 true
233 }
234 _ => false,
235 }
236}
237
238#[expect(
244 clippy::too_many_arguments,
245 reason = "the error path needs the task, agent, model and store context together"
246)]
247pub async fn handle_error(
248 task_id: &str,
249 agent_err: &agents::AgentError,
250 agent_name: &str,
251 agent_runner: &dyn agents::AgentRunner,
252 model_name: Option<&str>,
253 complexity: Option<&str>,
254 new_attempts: u32,
255 store: &Option<Arc<TaskStore>>,
256 repo: &str,
257) -> anyhow::Result<ErrorHandleResult> {
258 tracing::warn!(
259 task_id,
260 agent = agent_name,
261 error = %agent_err,
262 "agent error, attempting recovery"
263 );
264
265 let mut credit_recorded = false;
268 let (retryable, error_msg) = match agent_err {
269 agents::AgentError::RateLimit { message, .. } => {
270 let is_provider_returned_error = message
271 .to_ascii_lowercase()
272 .contains("provider returned error");
273 let mut billing_cycle_model_scoped = false;
293
294 if apply_vendor_reset_date(agent_name, model_name, message).await {
295 let reason = cooldown::detect_credit_exhaustion(message);
296 credit_recorded = reason.is_some();
297 billing_cycle_model_scoped =
298 reason == Some(CreditExhaustionReason::BillingCycleExhausted);
299 } else if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
300 credit_recorded = true;
301 if reason == CreditExhaustionReason::BillingCycleExhausted {
307 if let Some(model) = model_name {
308 cooldown::record_persistent_model_failure(agent_name, model).await;
309 billing_cycle_model_scoped = true;
310 } else {
311 cooldown::record_credit_exhaustion(agent_name, reason).await;
312 }
313 } else {
314 cooldown::record_credit_exhaustion(agent_name, reason).await;
315 }
316 } else if let (Some(model), Some(window_secs)) =
317 (model_name, cooldown::parse_relative_usage_window(message))
318 {
319 cooldown::set_model_cooldown(
327 agent_name,
328 model,
329 window_secs,
330 "rate_limit_usage_window",
331 )
332 .await;
333 tracing::info!(
334 task_id,
335 agent = agent_name,
336 model,
337 window_secs,
338 "relative usage window detected: applying model-specific cooldown"
339 );
340 } else {
341 cooldown::record_agent_failure_with_message(agent_name, message).await;
342 if let Some(model) = model_name {
344 if is_provider_returned_error {
345 cooldown::record_transient_model_failure(agent_name, model).await;
346 } else {
347 response::record_model_failure(agent_name, model).await;
348 }
349 }
350 }
351 let retryable = if billing_cycle_model_scoped {
355 response::RetryableError::ModelUnavailable
356 } else {
357 response::RetryableError::UsageLimit
358 };
359 (
360 retryable,
361 format!(
362 "{agent_name} rate limit: {}",
363 summarize_rate_limit_error(message)
364 ),
365 )
366 }
367 agents::AgentError::Auth { message } => {
368 let mut billing_cycle_model_scoped = false;
373
374 if apply_vendor_reset_date(agent_name, model_name, message).await {
375 let reason = cooldown::detect_credit_exhaustion(message);
376 credit_recorded = reason.is_some();
377 billing_cycle_model_scoped =
378 reason == Some(CreditExhaustionReason::BillingCycleExhausted);
379 } else if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
380 credit_recorded = true;
381 if reason == CreditExhaustionReason::BillingCycleExhausted {
384 if let Some(model) = model_name {
385 cooldown::record_persistent_model_failure(agent_name, model).await;
386 billing_cycle_model_scoped = true;
387 } else {
388 cooldown::record_credit_exhaustion(agent_name, reason).await;
389 }
390 } else {
391 cooldown::record_credit_exhaustion(agent_name, reason).await;
392 }
393 } else {
394 cooldown::record_agent_failure_with_message(agent_name, message).await;
395 if let Some(model) = model_name {
396 response::record_model_failure(agent_name, model).await;
397 }
398 }
399 let retryable = if billing_cycle_model_scoped {
402 response::RetryableError::ModelUnavailable
403 } else {
404 response::RetryableError::AuthError
405 };
406 (retryable, format!("{agent_name} auth error: {message}"))
407 }
408 agents::AgentError::Timeout { elapsed } => {
409 (
412 response::RetryableError::Timeout,
413 format!("{agent_name} timed out after {}s", elapsed.as_secs()),
414 )
415 }
416 agents::AgentError::Interrupted { message } => (
417 response::RetryableError::Interrupted,
418 format!("{agent_name} interrupted: {message}"),
419 ),
420 agents::AgentError::MissingTool { tool } => (
421 response::RetryableError::MissingTooling,
422 format!("missing tool: {tool}"),
423 ),
424 agents::AgentError::ModelUnavailable { model, message } => {
425 cooldown::record_model_unavailable(agent_name, model, message).await;
426
427 let agent_cooled = cooldown::is_agent_in_cooldown(agent_name);
430 let models = agent_runner.available_models();
431 let current_model = model_name.unwrap_or("");
432 let next_model = if agent_cooled {
433 None
434 } else {
435 models.iter().find(|m| {
436 m.as_str() != current_model
437 && m.as_str() != model
438 && !response::is_model_in_cooldown(agent_name, m)
439 })
440 };
441 if let Some(next) = next_model {
442 tracing::info!(task_id, model = %next, "retrying with different model");
443 let msg = format!("model {model} unavailable, trying {next}");
444 if let Err(e) = store::store_set_result(
445 store,
446 repo,
447 task_id,
448 &[
449 ("model", serde_json::json!(next.clone())),
450 ("last_error", serde_json::json!(msg)),
451 ],
452 )
453 .await
454 {
455 tracing::warn!(task_id, err = %e, "failed to write model failover to store");
456 }
457 response::wait_for_fallback_backoff(task_id, store, repo).await;
461 return Ok(ErrorHandleResult::EarlyReturn {
462 status: "routed".to_string(),
463 });
464 }
465
466 let is_simple = matches!(complexity, None | Some("simple"));
468 if is_simple
469 && let Some(result) = try_free_model_reroute(
470 task_id,
471 agent_name,
472 agent_runner,
473 &[current_model, model.as_str()],
474 true,
475 None,
476 true,
477 &format!("model {model} unavailable, trying free model"),
478 store,
479 repo,
480 )
481 .await
482 {
483 return Ok(result);
484 }
485
486 (
487 response::RetryableError::ModelUnavailable,
488 format!("model {model} unavailable"),
489 )
490 }
491 agents::AgentError::ContextOverflow { message } => {
492 let msg = format!("{agent_name} context overflow: {message}");
498 if let Err(e) = store::store_set_result(
499 store,
500 repo,
501 task_id,
502 &[("last_error", serde_json::json!(msg))],
503 )
504 .await
505 {
506 tracing::warn!(task_id, err = %e, "failed to write context_overflow last_error to store");
507 }
508 return Ok(ErrorHandleResult::EarlyReturn {
509 status: "needs_review".to_string(),
510 });
511 }
512 agents::AgentError::WaitingForInput { message } => {
513 let msg = format!("waiting for input: {message}");
515 if let Err(e) = store::store_set_result(
516 store,
517 repo,
518 task_id,
519 &[("last_error", serde_json::json!(msg))],
520 )
521 .await
522 {
523 tracing::warn!(task_id, err = %e, "failed to write waiting_for_input last_error to store");
524 }
525 return Ok(ErrorHandleResult::EarlyReturn {
526 status: "needs_review".to_string(),
527 });
528 }
529 agents::AgentError::PermissionDenied { message } => (
530 response::RetryableError::Failed,
531 format!("permission denied: {message}"),
532 ),
533 agents::AgentError::InvalidResponse { .. } => {
534 if let Some(model) = model_name {
537 response::record_model_failure(agent_name, model).await;
538 }
539 (
540 response::RetryableError::Failed,
541 format!("{agent_name} invalid response"),
542 )
543 }
544 agents::AgentError::AgentFailed { message } => {
545 if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
552 credit_recorded = true;
553 match model_name {
554 Some(model) if reason == CreditExhaustionReason::BillingCycleExhausted => {
555 cooldown::record_persistent_model_failure(agent_name, model).await;
556 }
557 _ => cooldown::record_credit_exhaustion(agent_name, reason).await,
558 }
559 } else {
560 if let Some(model) = model_name {
561 response::record_model_failure(agent_name, model).await;
562 }
563 cooldown::record_agent_failure_with_message(agent_name, message).await;
564 }
565 (
566 response::RetryableError::Failed,
567 format!("{agent_name} failed: {message}"),
568 )
569 }
570 agents::AgentError::NetworkError { message } => {
571 const MAX_NETWORK_RETRIES: u64 = 8;
577 let msg = format!("{agent_name} network error: {message}");
578 let retry_count = match store::store_increment(store, repo, task_id, "network_retries")
579 .await
580 {
581 Ok(count) => {
582 tracing::info!(
583 task_id,
584 retry_count = count,
585 agent = agent_name,
586 "network retry scheduled"
587 );
588 count
589 }
590 Err(e) => {
591 tracing::warn!(
592 task_id,
593 error = %e,
594 "failed to increment network_retries — escalating to needs_review"
595 );
596 return Ok(ErrorHandleResult::Continue {
597 status: "needs_review".to_string(),
598 error: format!("{agent_name} network error (store unavailable): {message}"),
599 });
600 }
601 };
602 if let Err(e) = store::store_set_result(
603 store,
604 repo,
605 task_id,
606 &[("last_error", serde_json::json!(msg))],
607 )
608 .await
609 {
610 tracing::warn!(task_id, err = %e, "failed to write network_error last_error to store");
611 }
612 if retry_count >= MAX_NETWORK_RETRIES {
613 tracing::warn!(
614 task_id,
615 retry_count,
616 agent = agent_name,
617 "network retries exhausted, escalating to needs_review"
618 );
619 return Ok(ErrorHandleResult::Continue {
620 status: "needs_review".to_string(),
621 error: msg,
622 });
623 }
624 response::wait_for_fallback_backoff(task_id, store, repo).await;
626 return Ok(ErrorHandleResult::EarlyReturn {
627 status: "routed".to_string(),
628 });
629 }
630 agents::AgentError::StaleSession { session_id } => {
631 (
635 response::RetryableError::Failed,
636 format!("{agent_name} stale session: {session_id}"),
637 )
638 }
639 agents::AgentError::ThinkingBlockConflict { message } => {
640 (
646 response::RetryableError::ModelUnavailable,
647 format!("{agent_name} thinking block conflict: {message}"),
648 )
649 }
650 agents::AgentError::Unknown { exit_code, message } => {
651 if *exit_code == 0
657 && (message.is_empty()
658 || message == "empty-output-exit0"
659 || message.starts_with("empty-output-exit0:"))
660 {
661 if let Some(m) = model_name {
662 cooldown::record_transient_model_failure(agent_name, m).await;
664 }
665 let current = model_name.unwrap_or("");
670 if let Some(result) = try_free_model_reroute(
671 task_id,
672 agent_name,
673 agent_runner,
674 &[current],
675 true,
676 Some("opencode"),
677 true,
678 "silent exit 0, retrying with free model",
679 store,
680 repo,
681 )
682 .await
683 {
684 return Ok(result);
685 }
686 }
687
688 let error_message = if *exit_code != 0 && message.contains("\"costUSD\":") {
691 format!(
693 "{agent_name} exited with non-zero status (cost telemetry in stdout, no error message)"
694 )
695 } else {
696 format!("{agent_name} exit {exit_code}: {message}")
697 };
698
699 (response::RetryableError::Failed, error_message)
700 }
701 };
702
703 if matches!(
705 retryable,
706 response::RetryableError::Timeout | response::RetryableError::Interrupted
707 ) && let Err(e) = store::store_set_result(
708 store,
709 repo,
710 task_id,
711 &[
712 ("agent", serde_json::json!("")),
713 ("last_error", serde_json::json!(error_msg.clone())),
714 ],
715 )
716 .await
717 {
718 tracing::warn!(task_id, err = %e, "failed to write timeout agent clear to store");
719 }
720
721 {
724 let error_type_str = match retryable {
725 response::RetryableError::UsageLimit => Some("rate"),
726 response::RetryableError::AuthError => Some("auth"),
727 response::RetryableError::Timeout | response::RetryableError::Interrupted => None,
728 _ => Some("error"),
729 };
730 if let Some(limit_type) = error_type_str
731 && let Some(s) = store
732 && let Err(e) = s
733 .record_rate_limit(agent_name, limit_type, Some(task_id))
734 .await
735 {
736 tracing::warn!(task_id, agent = %agent_name, err = %e, "failed to persist rate_limit event — router health check may miss this signal");
737 }
738 }
739
740 let chain = response::get_reroute_chain(task_id, store, repo).await;
742 let available: Vec<String> = engine::configured_agents()
743 .into_iter()
744 .filter(|a| cmd_cache::command_exists(a))
745 .collect();
746
747 let all_agents_tried = {
748 let chain_set: HashSet<&str> = if chain.is_empty() {
749 HashSet::new()
750 } else {
751 chain.split(',').collect()
752 };
753 !available
754 .iter()
755 .any(|a| a.as_str() != agent_name && !chain_set.contains(a.as_str()))
756 };
757
758 let is_simple = matches!(complexity, None | Some("simple"));
759 if all_agents_tried && is_simple {
760 if let Some(result) = try_free_model_reroute(
762 task_id,
763 agent_name,
764 agent_runner,
765 &[],
766 true, Some("opencode"),
768 false,
769 "all agents exhausted, trying free model",
770 store,
771 repo,
772 )
773 .await
774 {
775 return Ok(result);
776 }
777 }
778
779 let outcome = response::handle_failover(
780 task_id,
781 agent_name,
782 retryable,
783 &error_msg,
784 credit_recorded,
785 store,
786 repo,
787 )
788 .await;
789 match (outcome.status.as_str(), outcome.next_agent.as_deref()) {
790 ("new", Some(next)) => {
791 tracing::info!(task_id, to = next, "failover: re-routing to fallback agent");
792 }
793 ("new", None) => {
794 tracing::warn!(task_id, "failover exhausted (retryable), task reset to new");
795 }
796 ("blocked", _) => {
797 tracing::warn!(task_id, "failover exhausted (hard failure), task blocked");
798 }
799 _ => {}
800 }
801 let status = outcome.status;
802
803 response::store_failure_memory(
805 task_id,
806 new_attempts,
807 agent_name,
808 model_name,
809 &error_msg,
810 store,
811 repo,
812 )
813 .await;
814
815 Ok(ErrorHandleResult::Continue {
816 status,
817 error: error_msg,
818 })
819}
820
821#[cfg(test)]
822mod tests {
823 use std::time::Duration;
824
825 use super::*;
826 use crate::engine::runner::agents::StreamEvent;
827 use crate::engine::runner::agents::{AgentError, AgentRunner, ParsedResponse, PermissionRules};
828 use crate::parser::AgentResponse;
829 use chrono::Datelike;
830 use serial_test::serial;
831 use tokio::process::Command;
832
833 struct MockRunner {
834 free: Vec<String>,
835 }
836
837 struct MockRunnerWithModels {
838 models: Vec<String>,
839 }
840
841 impl AgentRunner for MockRunnerWithModels {
842 fn stream_events(&self, _event: &serde_json::Value) -> Vec<StreamEvent> {
843 vec![]
844 }
845
846 fn name(&self) -> &'static str {
847 "opencode"
848 }
849
850 fn build_command(
851 &self,
852 _model: Option<&str>,
853 _timeout_cmd: &str,
854 _sys_file: &str,
855 _msg_file: &str,
856 _permissions: &PermissionRules,
857 ) -> String {
858 String::new()
859 }
860
861 fn parse_response(&self, _raw: &str) -> Result<ParsedResponse, AgentError> {
862 Ok(ParsedResponse {
863 response: AgentResponse {
864 status: "done".to_string(),
865 summary: String::new(),
866 accomplished: vec![],
867 remaining: vec![],
868 files: vec![],
869 error: None,
870 input_tokens: None,
871 output_tokens: None,
872 learnings: vec![],
873 delegations: vec![],
874 screenshots: vec![],
875 screenshot_note: None,
876 },
877 input_tokens: None,
878 output_tokens: None,
879 cost_usd: None,
880 duration_ms: None,
881 })
882 }
883
884 fn classify_error(&self, _exit_code: i32, _stdout: &str, _stderr: &str) -> AgentError {
885 AgentError::Unknown {
886 exit_code: 0,
887 message: String::new(),
888 }
889 }
890
891 fn available_models(&self) -> Vec<String> {
892 self.models.clone()
893 }
894
895 fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
896 anyhow::bail!("not implemented")
897 }
898 }
899
900 impl AgentRunner for MockRunner {
901 fn stream_events(&self, _event: &serde_json::Value) -> Vec<StreamEvent> {
902 vec![]
903 }
904
905 fn name(&self) -> &'static str {
906 "opencode"
907 }
908
909 fn build_command(
910 &self,
911 _model: Option<&str>,
912 _timeout_cmd: &str,
913 _sys_file: &str,
914 _msg_file: &str,
915 _permissions: &PermissionRules,
916 ) -> String {
917 String::new()
918 }
919
920 fn parse_response(&self, _raw: &str) -> Result<ParsedResponse, AgentError> {
921 Ok(ParsedResponse {
922 response: AgentResponse {
923 status: "done".to_string(),
924 summary: String::new(),
925 accomplished: vec![],
926 remaining: vec![],
927 files: vec![],
928 error: None,
929 input_tokens: None,
930 output_tokens: None,
931 learnings: vec![],
932 delegations: vec![],
933 screenshots: vec![],
934 screenshot_note: None,
935 },
936 input_tokens: None,
937 output_tokens: None,
938 cost_usd: None,
939 duration_ms: None,
940 })
941 }
942
943 fn classify_error(&self, _exit_code: i32, _stdout: &str, _stderr: &str) -> AgentError {
944 AgentError::Unknown {
945 exit_code: 0,
946 message: String::new(),
947 }
948 }
949
950 fn free_models(&self) -> Vec<String> {
951 self.free.clone()
952 }
953
954 fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
955 anyhow::bail!("not implemented")
956 }
957 }
958
959 #[serial(cooldown_state)]
965 #[tokio::test]
966 async fn rate_limit_sets_agent_cooldown_immediately() {
967 cooldown::reset_global_state().await;
968 let runner = MockRunner { free: vec![] };
969 let agent = "test-agent-1371-early-cooldown";
970
971 assert!(
973 !cooldown::is_agent_in_cooldown(agent),
974 "agent should not be in cooldown before handle_error"
975 );
976
977 let err = AgentError::RateLimit {
978 message: "429 Too Many Requests".to_string(),
979 };
980
981 let _result = handle_error(
983 "test-1371-a",
984 &err,
985 agent,
986 &runner,
987 Some("sonnet"),
988 Some("medium"),
989 1,
990 &None,
991 "owner/repo",
992 )
993 .await
994 .unwrap();
995
996 assert!(
998 cooldown::is_agent_in_cooldown(agent),
999 "agent should be in cooldown immediately after handle_error for RateLimit"
1000 );
1001 }
1002
1003 #[serial(cooldown_state)]
1010 #[tokio::test]
1011 async fn rate_limit_sets_both_agent_and_model_cooldowns() {
1012 cooldown::reset_global_state().await;
1013 let runner = MockRunner { free: vec![] };
1014 let agent = "test-agent-2153-model-cooldown";
1015 let model = "opencode/github-copilot/qwen3.6-plus-free";
1016
1017 assert!(
1019 !cooldown::is_agent_in_cooldown(agent),
1020 "agent should not be in cooldown before handle_error"
1021 );
1022 assert!(
1023 !cooldown::is_model_in_cooldown(agent, model),
1024 "model should not be in cooldown before handle_error"
1025 );
1026
1027 let err = AgentError::RateLimit {
1028 message: "Upstream error from Alibaba: Request rate increased too quickly.".to_string(),
1029 };
1030
1031 let _result = handle_error(
1032 "test-2153-a",
1033 &err,
1034 agent,
1035 &runner,
1036 Some(model),
1037 Some("simple"),
1038 1,
1039 &None,
1040 "owner/repo",
1041 )
1042 .await
1043 .unwrap();
1044
1045 assert!(
1047 cooldown::is_agent_in_cooldown(agent),
1048 "agent should be in agent-level cooldown after RateLimit"
1049 );
1050 assert!(
1053 cooldown::is_model_in_cooldown(agent, model),
1054 "model should be in model-level cooldown after RateLimit (fixes issue #2153)"
1055 );
1056 }
1057
1058 #[serial(cooldown_state)]
1060 #[tokio::test]
1061 async fn rate_limit_cooldown_is_model_specific() {
1062 cooldown::reset_global_state().await;
1063 let runner = MockRunner { free: vec![] };
1064 let agent = "test-agent-2153-model-specific";
1065 let cooled_model = "opencode/github-copilot/qwen3.6-plus-free";
1066 let other_model = "opencode/minimax-m2.5-free";
1067
1068 let err = AgentError::RateLimit {
1069 message: "rate limit exceeded".to_string(),
1070 };
1071
1072 let _result = handle_error(
1073 "test-2153-b",
1074 &err,
1075 agent,
1076 &runner,
1077 Some(cooled_model),
1078 Some("simple"),
1079 1,
1080 &None,
1081 "owner/repo",
1082 )
1083 .await
1084 .unwrap();
1085
1086 assert!(
1088 cooldown::is_model_in_cooldown(agent, cooled_model),
1089 "rate-limited model should be in cooldown"
1090 );
1091 assert!(
1093 !cooldown::is_model_in_cooldown(agent, other_model),
1094 "other models should not be cooled by a single model's rate limit"
1095 );
1096 }
1097
1098 #[serial(cooldown_state)]
1109 #[tokio::test]
1110 async fn credit_exhaustion_does_not_bump_generic_failure_count() {
1111 cooldown::reset_global_state().await;
1112 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1113 cooldown::init_cooldown_store(Arc::clone(&store)).await;
1114
1115 let runner = MockRunner { free: vec![] };
1116 let agent = "test-agent-3652-credit-no-bump";
1117
1118 let err = AgentError::RateLimit {
1119 message: "out of credits, please recharge your balance".to_string(),
1120 };
1121
1122 let _result = handle_error(
1123 "test-3652-a",
1124 &err,
1125 agent,
1126 &runner,
1127 None,
1128 Some("simple"),
1129 1,
1130 &Some(Arc::clone(&store)),
1131 "owner/repo",
1132 )
1133 .await
1134 .unwrap();
1135
1136 let credit_count = store
1137 .kv_get(&format!("credit_failure_count:{agent}"))
1138 .await
1139 .unwrap();
1140 assert!(
1141 credit_count.is_some(),
1142 "credit_failure_count should be incremented by the credit error"
1143 );
1144 let generic_count = store
1145 .kv_get(&format!("failure_count:{agent}"))
1146 .await
1147 .unwrap();
1148 assert!(
1149 generic_count.is_none(),
1150 "failure_count must not be bumped by credit errors, got {generic_count:?}"
1151 );
1152 }
1153
1154 #[serial(cooldown_state)]
1157 #[tokio::test]
1158 async fn two_timeouts_leave_model_routable() {
1159 cooldown::reset_global_state().await;
1160 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1161 cooldown::init_cooldown_store(Arc::clone(&store)).await;
1162
1163 let runner = MockRunner { free: vec![] };
1164 let agent = "opencode";
1165 let model = "opencode/nemotron-3-ultra-free";
1166
1167 let err = AgentError::Timeout {
1168 elapsed: Duration::from_mins(30),
1169 };
1170
1171 for i in 1..=2 {
1172 let _result = handle_error(
1173 &format!("test-3876-timeout-{i}"),
1174 &err,
1175 agent,
1176 &runner,
1177 Some(model),
1178 Some("simple"),
1179 i,
1180 &Some(Arc::clone(&store)),
1181 "owner/repo",
1182 )
1183 .await
1184 .unwrap();
1185 }
1186
1187 assert!(
1188 !cooldown::is_model_in_cooldown(agent, model),
1189 "timeouts must not cool the model"
1190 );
1191 assert!(
1192 !cooldown::is_agent_in_cooldown(agent),
1193 "timeouts must not cool the agent"
1194 );
1195 let counts = store.recent_rate_limit_counts(6).await.unwrap();
1196 assert_eq!(
1197 counts.get(agent).copied().unwrap_or(0),
1198 0,
1199 "timeouts must not write a rate_limits row"
1200 );
1201 }
1202
1203 #[serial(cooldown_state)]
1206 #[tokio::test]
1207 async fn interrupted_run_does_not_write_rate_limit() {
1208 cooldown::reset_global_state().await;
1209 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1210 cooldown::init_cooldown_store(Arc::clone(&store)).await;
1211
1212 let runner = MockRunner { free: vec![] };
1213 let agent = "claude";
1214 let model = "sonnet";
1215 let err = AgentError::Interrupted {
1216 message: "exited without a result envelope".to_string(),
1217 };
1218
1219 let _result = handle_error(
1220 "test-3876-interrupted",
1221 &err,
1222 agent,
1223 &runner,
1224 Some(model),
1225 Some("medium"),
1226 1,
1227 &Some(Arc::clone(&store)),
1228 "owner/repo",
1229 )
1230 .await
1231 .unwrap();
1232
1233 assert!(
1234 !cooldown::is_model_in_cooldown(agent, model),
1235 "interrupted runs must not cool the model"
1236 );
1237 let counts = store.recent_rate_limit_counts(6).await.unwrap();
1238 assert_eq!(
1239 counts.get(agent).copied().unwrap_or(0),
1240 0,
1241 "interrupted runs must not write a rate_limits row"
1242 );
1243 }
1244
1245 #[serial(cooldown_state)]
1258 #[tokio::test]
1259 async fn rate_limit_with_relative_window_sets_window_length_model_cooldown() {
1260 cooldown::reset_global_state().await;
1261 let runner = MockRunner { free: vec![] };
1262 let agent = "test-agent-3580-kimi";
1263 let model = "opus";
1264 let key = format!("{agent}:{model}");
1265
1266 let err = AgentError::RateLimit {
1267 message: "You've reached your weekly (7-day) usage limit. Your quota will reset when the current 7-day window expires.".to_string(),
1268 };
1269
1270 let _result = handle_error(
1271 "test-3580-a",
1272 &err,
1273 agent,
1274 &runner,
1275 Some(model),
1276 Some("medium"),
1277 1,
1278 &None,
1279 "owner/repo",
1280 )
1281 .await
1282 .unwrap();
1283
1284 let now = chrono::Utc::now().timestamp();
1285 let until = cooldown::cooldown_until(&key)
1286 .expect("relative usage window should set model cooldown");
1287 let remaining = until.saturating_sub(now);
1288
1289 assert!(
1290 remaining >= 7 * 86400 - 30,
1291 "expected ~7-day model cooldown from the parsed window, got {remaining}s"
1292 );
1293 assert_eq!(
1294 cooldown::cooldown_reason(&key).as_deref(),
1295 Some("rate_limit_usage_window"),
1296 "relative usage-window cooldown must be labeled as a rate limit, not silence"
1297 );
1298 }
1299
1300 #[serial(cooldown_state)]
1301 #[tokio::test]
1302 async fn provider_returned_error_sets_short_model_cooldown() {
1303 cooldown::reset_global_state().await;
1304 let runner = MockRunner { free: vec![] };
1305 let agent = "opencode";
1306 let model = "opencode/nemotron-3-super-free";
1307 let key = format!("{agent}:{model}");
1308
1309 let err = AgentError::RateLimit {
1310 message: "Provider returned error".to_string(),
1311 };
1312
1313 let _result = handle_error(
1314 "test-2478-provider-cooldown",
1315 &err,
1316 agent,
1317 &runner,
1318 Some(model),
1319 Some("simple"),
1320 1,
1321 &None,
1322 "owner/repo",
1323 )
1324 .await
1325 .unwrap();
1326
1327 let now = chrono::Utc::now().timestamp();
1328 let until =
1329 cooldown::cooldown_until(&key).expect("provider error should set model cooldown");
1330 let remaining = until.saturating_sub(now);
1331
1332 assert!(
1333 remaining <= cooldown::TRANSIENT_MODEL_BACKOFF_BASE_SECS,
1334 "expected a 5 minute model cooldown, got {remaining}s"
1335 );
1336 }
1337
1338 #[serial(cooldown_state)]
1339 #[tokio::test]
1340 async fn model_unavailable_sets_persistent_model_cooldown_for_codex_style() {
1341 cooldown::reset_global_state().await;
1342 let runner = MockRunner { free: vec![] };
1343 let agent = "codex";
1344 let model = "gpt-5.3-codex";
1345
1346 let err = AgentError::ModelUnavailable {
1348 model: model.to_string(),
1349 message: "{\"detail\":\"The 'gpt-5.3-codex' model is not supported\"}".to_string(),
1350 };
1351
1352 let _result = handle_error(
1353 "test-codex-model-unsupported",
1354 &err,
1355 agent,
1356 &runner,
1357 Some(model),
1358 Some("simple"),
1359 1,
1360 &None,
1361 "owner/repo",
1362 )
1363 .await
1364 .unwrap();
1365
1366 let key = format!("{agent}:{model}");
1368 let now = chrono::Utc::now().timestamp();
1369 let until = cooldown::cooldown_until(&key).expect("model cooldown should be set");
1370 let remaining = until.saturating_sub(now);
1371
1372 assert!(
1373 remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
1374 "expected persistent model cooldown (~4h) for codex model-unavailable, got {remaining}s"
1375 );
1376 }
1377
1378 async fn vendor_reset_remaining(agent: &str, model: &str, message: &str) -> i64 {
1379 cooldown::reset_global_state().await;
1380 let runner = MockRunner { free: vec![] };
1381 let err = AgentError::RateLimit {
1382 message: message.to_string(),
1383 };
1384 let _ = handle_error(
1385 "test-3649",
1386 &err,
1387 agent,
1388 &runner,
1389 Some(model),
1390 Some("simple"),
1391 1,
1392 &None,
1393 "owner/repo",
1394 )
1395 .await
1396 .unwrap();
1397 let until = cooldown::cooldown_until(&format!("{agent}:{model}"))
1398 .expect("model cooldown should be set");
1399 until - chrono::Utc::now().timestamp()
1400 }
1401
1402 #[serial(cooldown_state)]
1403 #[tokio::test]
1404 async fn vendor_reset_date_overrides_weekly_window() {
1405 let t = chrono::Local::now() + chrono::Duration::days(2);
1406 let msg = format!(
1407 "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
1408 t.format("%b"),
1409 t.day()
1410 );
1411 let remaining = vendor_reset_remaining("claude", "sonnet", &msg).await;
1412 assert!(
1413 remaining > 3600 && remaining < 3 * 86400,
1414 "expected ~2d cooldown, got {remaining}s"
1415 );
1416 }
1417
1418 #[serial(cooldown_state)]
1419 #[tokio::test]
1420 async fn vendor_reset_date_overrides_billing_cycle_backoff() {
1421 let t = chrono::Local::now() + chrono::Duration::days(3);
1422 let msg = format!(
1423 "You've hit your usage limit. Upgrade to Pro (https://x), or try again at {}.",
1424 t.format("%b %d, %Y %I:%M %p")
1425 );
1426 let remaining = vendor_reset_remaining("codex", "gpt-5.4", &msg).await;
1427 assert!(
1428 remaining > 2 * 86400 + 80000 - 3600 && remaining < 4 * 86400,
1429 "expected ~3d cooldown, got {remaining}s"
1430 );
1431 }
1432
1433 #[serial(cooldown_state)]
1434 #[tokio::test]
1435 async fn agent_failed_insufficient_balance_sets_credit_cooldown() {
1436 cooldown::reset_global_state().await;
1437 let runner = MockRunner { free: vec![] };
1438 let agent = "test-agent-3642-insufficient-balance";
1439
1440 assert!(
1442 !cooldown::is_agent_in_cooldown(agent),
1443 "agent should not be in cooldown before handle_error"
1444 );
1445
1446 let err = agents::AgentError::AgentFailed {
1447 message: "402 Payment Required: insufficient balance".to_string(),
1448 };
1449
1450 let _result = handle_error(
1452 "test-3642-a",
1453 &err,
1454 agent,
1455 &runner,
1456 Some("sonnet"),
1457 Some("medium"),
1458 1,
1459 &None,
1460 "owner/repo",
1461 )
1462 .await
1463 .unwrap();
1464
1465 assert!(
1470 cooldown::is_agent_in_cooldown(agent),
1471 "agent should be in cooldown after handle_error for AgentFailed with insufficient balance"
1472 );
1473
1474 let until = cooldown::cooldown_until(agent).expect("agent cooldown should be active");
1475 let remaining = until - chrono::Utc::now().timestamp();
1476 assert!(
1477 remaining >= cooldown::CREDIT_BACKOFF_BASE_SECS - 30,
1478 "credit cooldown should be at least the 1h base, got {remaining}s"
1479 );
1480 }
1481
1482 #[serial(cooldown_state)]
1483 #[tokio::test]
1484 async fn silent_exit0_retries_free_model_for_simple_complexity() {
1485 cooldown::reset_global_state().await;
1486 let runner = MockRunner {
1487 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1488 };
1489 let err = AgentError::Unknown {
1490 exit_code: 0,
1491 message: String::new(),
1492 };
1493
1494 let result = handle_error(
1496 "test-1112-a",
1497 &err,
1498 "opencode",
1499 &runner,
1500 Some("opencode/gpt-5.4-mini"),
1501 Some("simple"),
1502 1,
1503 &None,
1504 "owner/repo",
1505 )
1506 .await
1507 .unwrap();
1508
1509 assert!(
1510 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1511 "expected EarlyReturn{{status: new}} for simple complexity — free model should be tried first"
1512 );
1513 }
1514
1515 #[serial(cooldown_state)]
1516 #[tokio::test]
1517 async fn silent_exit0_retries_free_model_when_complexity_unknown() {
1518 cooldown::reset_global_state().await;
1519 let runner = MockRunner {
1520 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1521 };
1522 let err = AgentError::Unknown {
1523 exit_code: 0,
1524 message: String::new(),
1525 };
1526
1527 let result = handle_error(
1529 "test-1112-a2",
1530 &err,
1531 "opencode",
1532 &runner,
1533 Some("opencode/gpt-5.4-mini"),
1534 None,
1535 1,
1536 &None,
1537 "owner/repo",
1538 )
1539 .await
1540 .unwrap();
1541
1542 assert!(
1543 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1544 "expected EarlyReturn{{status: new}} for unknown complexity — free model should be tried first"
1545 );
1546 }
1547
1548 #[serial(cooldown_state)]
1549 #[tokio::test]
1550 async fn silent_exit0_retries_free_model_for_medium_complexity() {
1551 cooldown::reset_global_state().await;
1552 let runner = MockRunner {
1553 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1554 };
1555 let err = AgentError::Unknown {
1556 exit_code: 0,
1557 message: String::new(),
1558 };
1559
1560 let result = handle_error(
1562 "test-1195-medium",
1563 &err,
1564 "opencode",
1565 &runner,
1566 Some("opencode/copilot-gpt-5.4-mini"),
1567 Some("medium"),
1568 1,
1569 &None,
1570 "owner/repo",
1571 )
1572 .await
1573 .unwrap();
1574
1575 assert!(
1576 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1577 "expected EarlyReturn{{status: new}} for medium complexity — free model should be tried before failover"
1578 );
1579 }
1580
1581 #[serial(cooldown_state)]
1582 #[tokio::test]
1583 async fn silent_exit0_retries_free_model_for_complex_complexity() {
1584 cooldown::reset_global_state().await;
1585 let runner = MockRunner {
1586 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1587 };
1588 let err = AgentError::Unknown {
1589 exit_code: 0,
1590 message: String::new(),
1591 };
1592
1593 let result = handle_error(
1595 "test-1195-complex",
1596 &err,
1597 "opencode",
1598 &runner,
1599 Some("opencode/copilot-gpt-5.4-mini"),
1600 Some("complex"),
1601 1,
1602 &None,
1603 "owner/repo",
1604 )
1605 .await
1606 .unwrap();
1607
1608 assert!(
1609 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1610 "expected EarlyReturn{{status: new}} for complex complexity — free model should be tried before failover"
1611 );
1612 }
1613
1614 #[serial(cooldown_state)]
1617 #[tokio::test]
1618 async fn context_overflow_escalates_to_needs_review_immediately() {
1619 cooldown::reset_global_state().await;
1620 let runner = MockRunner { free: vec![] };
1621 let err = AgentError::ContextOverflow {
1622 message: "prompt is too long: 250000 tokens, max 200000".to_string(),
1623 };
1624
1625 let result = handle_error(
1626 "test-2129-a",
1627 &err,
1628 "claude",
1629 &runner,
1630 Some("sonnet"),
1631 Some("medium"),
1632 1,
1633 &None,
1634 "owner/repo",
1635 )
1636 .await
1637 .unwrap();
1638
1639 assert!(
1640 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "needs_review"),
1641 "context overflow must escalate directly to needs_review without cycling through fallback agents"
1642 );
1643 }
1644
1645 #[serial(cooldown_state)]
1648 #[tokio::test]
1649 async fn free_model_reroute_succeeds_without_store() {
1650 cooldown::reset_global_state().await;
1651 let runner = MockRunner {
1652 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1653 };
1654 let err = AgentError::Unknown {
1655 exit_code: 0,
1656 message: String::new(),
1657 };
1658
1659 let result = handle_error(
1661 "test-2293-a",
1662 &err,
1663 "opencode",
1664 &runner,
1665 Some("opencode/gpt-5.4-mini"),
1666 Some("simple"),
1667 1,
1668 &None,
1669 "owner/repo",
1670 )
1671 .await
1672 .unwrap();
1673
1674 assert!(
1675 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1676 "expected EarlyReturn{{status: new}} — store_set_result with None store should not prevent reroute"
1677 );
1678 }
1679
1680 #[serial(cooldown_state)]
1681 #[tokio::test]
1682 async fn silent_exit0_falls_through_when_no_free_models() {
1683 cooldown::reset_global_state().await;
1684 let runner = MockRunner { free: vec![] };
1685 let err = AgentError::Unknown {
1686 exit_code: 0,
1687 message: String::new(),
1688 };
1689
1690 let result = handle_error(
1691 "test-1112-b",
1692 &err,
1693 "opencode",
1694 &runner,
1695 Some("opencode/gpt-5.4-mini"),
1696 Some("simple"),
1697 1,
1698 &None,
1699 "owner/repo",
1700 )
1701 .await
1702 .unwrap();
1703
1704 assert!(
1705 matches!(result, ErrorHandleResult::Continue { .. }),
1706 "expected Continue (fallthrough to failover) when no free models available"
1707 );
1708 }
1709
1710 #[serial(cooldown_state)]
1716 #[tokio::test]
1717 async fn invalid_response_sets_model_cooldown() {
1718 cooldown::reset_global_state().await;
1719 let runner = MockRunner { free: vec![] };
1720 let agent = "opencode";
1721 let model = "opencode/nemotron-3-super-free";
1722
1723 assert!(
1725 !cooldown::is_model_in_cooldown(agent, model),
1726 "model should not be in cooldown before handle_error"
1727 );
1728
1729 let err = AgentError::InvalidResponse {
1730 raw: "invalid json output".to_string(),
1731 };
1732
1733 let _result = handle_error(
1734 "test-2750-invalid-response",
1735 &err,
1736 agent,
1737 &runner,
1738 Some(model),
1739 Some("simple"),
1740 1,
1741 &None,
1742 "owner/repo",
1743 )
1744 .await
1745 .unwrap();
1746
1747 assert!(
1749 cooldown::is_model_in_cooldown(agent, model),
1750 "model should be in cooldown after InvalidResponse (fixes issue #2750)"
1751 );
1752 }
1753
1754 #[serial(cooldown_state)]
1761 #[tokio::test]
1762 async fn billing_cycle_exhausted_with_known_model_sets_only_model_cooldown() {
1763 cooldown::reset_global_state().await;
1764 let runner = MockRunner { free: vec![] };
1765 let agent = "codex-3382-billing-cycle";
1766 let model = "gpt-5.5";
1767
1768 let err = AgentError::RateLimit {
1769 message: "rate limit: You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro).".to_string(),
1770 };
1771
1772 let _result = handle_error(
1773 "test-3382-a",
1774 &err,
1775 agent,
1776 &runner,
1777 Some(model),
1778 Some("medium"),
1779 1,
1780 &None,
1781 "owner/repo",
1782 )
1783 .await
1784 .unwrap();
1785
1786 assert!(
1788 !cooldown::is_agent_in_cooldown(agent),
1789 "agent must NOT be in agent-level cooldown when BillingCycleExhausted has a known model (issue #3382)"
1790 );
1791 assert!(
1793 cooldown::is_model_in_cooldown(agent, model),
1794 "model must be in cooldown after BillingCycleExhausted (issue #3382)"
1795 );
1796 assert!(
1798 !cooldown::is_model_in_cooldown(agent, "gpt-5.4"),
1799 "other models on the same agent must not be cooled by a single model's billing exhaustion"
1800 );
1801 }
1802
1803 #[serial(cooldown_state)]
1804 #[tokio::test]
1805 async fn summarize_rate_limit_error_extracts_api_retry_info() {
1806 cooldown::reset_global_state().await;
1807 let raw_output = r#"some log lines
1808{"type":"system","subtype":"api_retry","attempt":6,"max_retries":10,"retry_delay_ms":17266.789,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1809{"type":"system","subtype":"api_retry","attempt":7,"max_retries":10,"retry_delay_ms":35162.66,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1810more logs"#;
1811
1812 let summarized = super::summarize_rate_limit_error(raw_output);
1813 assert_eq!(summarized, "status=429 after 7 attempts (last delay 35s)");
1814 }
1815
1816 #[serial(cooldown_state)]
1817 #[tokio::test]
1818 async fn summarize_rate_limit_error_returns_original_when_no_api_retry() {
1819 cooldown::reset_global_state().await;
1820 let raw_output = "some regular error message without api_retry JSON";
1821
1822 let summarized = super::summarize_rate_limit_error(raw_output);
1823 assert_eq!(summarized, raw_output);
1824 }
1825
1826 #[serial(cooldown_state)]
1827 #[tokio::test]
1828 async fn summarize_rate_limit_error_handles_malformed_json_gracefully() {
1829 cooldown::reset_global_state().await;
1830 let raw_output = r#"some logs
1831{"type":"system","subtype":"api_retry","attempt":"not_a_number","retry_delay_ms":invalid,"error_status":429}
1832more logs"#;
1833
1834 let summarized = super::summarize_rate_limit_error(raw_output);
1835 assert!(summarized.starts_with("status="));
1837 assert!(summarized.contains("after"));
1838 assert!(summarized.contains("attempts"));
1839 assert!(summarized.contains("(last delay"));
1840 }
1841
1842 #[serial(cooldown_state)]
1843 #[tokio::test]
1844 async fn summarize_rate_limit_error_works_with_single_line() {
1845 cooldown::reset_global_state().await;
1846 let raw_output = r#"{"type":"system","subtype":"api_retry","attempt":3,"max_retries":10,"retry_delay_ms":5000,"error_status":429,"error":"rate_limit"}"#;
1847
1848 let summarized = super::summarize_rate_limit_error(raw_output);
1849 assert_eq!(summarized, "status=429 after 3 attempts (last delay 5s)");
1850 }
1851
1852 #[serial(cooldown_state)]
1856 #[tokio::test]
1857 async fn extract_json_field_handles_quoted_strings() {
1858 cooldown::reset_global_state().await;
1859 let json = r#"{"error_status":429,"attempt":7,"retry_delay_ms":35162.66,"error":"rate_limit","session_id":"331c099b-..."}"#;
1860
1861 assert_eq!(
1862 super::extract_json_field(json, "\"error\":"),
1863 Some("rate_limit".to_string()),
1864 "quoted string field mid-object should be extracted correctly"
1865 );
1866 assert_eq!(
1867 super::extract_json_field(json, "\"session_id\":"),
1868 Some("331c099b-...".to_string()),
1869 "quoted string field at end of object (no trailing comma) should be extracted"
1870 );
1871 assert_eq!(
1873 super::extract_json_field(json, "\"error_status\":"),
1874 Some("429".to_string())
1875 );
1876 assert_eq!(
1877 super::extract_json_field(json, "\"attempt\":"),
1878 Some("7".to_string())
1879 );
1880 assert_eq!(
1881 super::extract_json_field(json, "\"retry_delay_ms\":"),
1882 Some("35162.66".to_string())
1883 );
1884 }
1885
1886 #[serial(cooldown_state)]
1889 #[tokio::test]
1890 async fn summarize_rate_limit_error_handles_truncated_fragment_with_subtype() {
1891 cooldown::reset_global_state().await;
1892 let raw_output = r#"some log context
1894,"subtype":"api_retry","attempt":5,"retry_delay_ms":20000,"error_status":429,"error":"rate_limit"
1895more context"#;
1896
1897 let summarized = super::summarize_rate_limit_error(raw_output);
1898 assert!(
1899 !summarized.contains("subtype"),
1900 "summarized output must not contain raw 'subtype' JSON fragment"
1901 );
1902 assert!(
1903 !summarized.contains('{'),
1904 "summarized output must not contain JSON braces"
1905 );
1906 assert!(
1907 summarized.starts_with("status="),
1908 "summarized output should start with 'status=' — got: {summarized}"
1909 );
1910 }
1914
1915 #[serial(cooldown_state)]
1917 #[tokio::test]
1918 async fn summarize_rate_limit_error_handles_fragment_with_extractable_fields() {
1919 cooldown::reset_global_state().await;
1920 let raw_output = r#"log output
1921,"subtype":"api_retry","attempt":5,"retry_delay_ms":20000,"error_status":429,"error":"rate_limit","session_id":"abc123","max_retries":10}
1922rest of output"#;
1923
1924 let summarized = super::summarize_rate_limit_error(raw_output);
1925 assert_eq!(summarized, "status=429 after 5 attempts (last delay 20s)");
1926 }
1927
1928 #[serial(cooldown_state)]
1930 #[tokio::test]
1931 async fn summarize_rate_limit_error_complete_json_still_works() {
1932 cooldown::reset_global_state().await;
1933 let raw_output = r#"some log lines
1934{"type":"system","subtype":"api_retry","attempt":6,"max_retries":10,"retry_delay_ms":17266.789,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1935{"type":"system","subtype":"api_retry","attempt":7,"max_retries":10,"retry_delay_ms":35162.66,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1936more logs"#;
1937
1938 let summarized = super::summarize_rate_limit_error(raw_output);
1939 assert_eq!(summarized, "status=429 after 7 attempts (last delay 35s)");
1940 }
1941
1942 #[serial(cooldown_state)]
1944 #[tokio::test]
1945 async fn summarize_rate_limit_error_no_api_retry_no_raw_passthrough() {
1946 cooldown::reset_global_state().await;
1947 let raw_output = "some regular error message without api_retry JSON that is quite long and should be truncated";
1948
1949 let summarized = super::summarize_rate_limit_error(raw_output);
1950 assert!(
1952 summarized.len() <= 140,
1953 "expected truncated first-line, got: {summarized}"
1954 );
1955 }
1956
1957 #[serial(cooldown_state)]
1959 #[tokio::test]
1960 async fn summarize_rate_limit_error_no_api_retry_short() {
1961 cooldown::reset_global_state().await;
1962 let raw_output = "rate limit exceeded";
1963
1964 let summarized = super::summarize_rate_limit_error(raw_output);
1965 assert_eq!(summarized, raw_output);
1966 }
1967
1968 #[serial(cooldown_state)]
1974 #[tokio::test]
1975 async fn model_not_found_applies_persistent_cooldown() {
1976 cooldown::reset_global_state().await;
1977 let runner = MockRunner { free: vec![] };
1978 let agent = "opencode-2941-not-found";
1979 let model = "github-copilot/claude-opus-4.6";
1980 let key = format!("{agent}:{model}");
1981
1982 let err = AgentError::ModelUnavailable {
1983 model: model.to_string(),
1984 message: "Model not found: github-copilot/claude-opus-4.6".to_string(),
1985 };
1986
1987 let _result = handle_error(
1988 "test-2941-a",
1989 &err,
1990 agent,
1991 &runner,
1992 Some(model),
1993 Some("complex"),
1994 1,
1995 &None,
1996 "owner/repo",
1997 )
1998 .await
1999 .unwrap();
2000
2001 let now = chrono::Utc::now().timestamp();
2002 let until =
2003 cooldown::cooldown_until(&key).expect("model not found should set model cooldown");
2004 let remaining = until.saturating_sub(now);
2005
2006 assert!(
2009 remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2010 "expected ~4h (persistent) cooldown for 'Model not found', got {remaining}s — \
2011 may have applied transient 5min cooldown instead"
2012 );
2013 }
2014
2015 #[serial(cooldown_state)]
2022 #[tokio::test]
2023 async fn model_not_supported_applies_persistent_cooldown() {
2024 cooldown::reset_global_state().await;
2025 let runner = MockRunner { free: vec![] };
2026 let agent = "codex-3215-not-supported";
2027 let model = "gpt-5.2-codex";
2028 let key = format!("{agent}:{model}");
2029
2030 let err = AgentError::ModelUnavailable {
2031 model: model.to_string(),
2032 message: "The 'gpt-5.2-codex' model is not supported when using Codex with a ChatGPT account.".to_string(),
2033 };
2034
2035 let _result = handle_error(
2036 "test-3215-a",
2037 &err,
2038 agent,
2039 &runner,
2040 Some(model),
2041 Some("medium"),
2042 1,
2043 &None,
2044 "owner/repo",
2045 )
2046 .await
2047 .unwrap();
2048
2049 let now = chrono::Utc::now().timestamp();
2050 let until =
2051 cooldown::cooldown_until(&key).expect("model not supported should set model cooldown");
2052 let remaining = until.saturating_sub(now);
2053
2054 assert!(
2055 remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2056 "expected ~4h (persistent) cooldown for 'not supported', got {remaining}s — \
2057 may have applied transient 5min cooldown instead (issue #3215)"
2058 );
2059 }
2060
2061 #[serial(cooldown_state)]
2062 #[tokio::test]
2063 async fn model_not_available_in_country_applies_persistent_cooldown() {
2064 cooldown::reset_global_state().await;
2065 let runner = MockRunner { free: vec![] };
2066 let agent = "opencode-3650-country";
2067 let model = "some-region-locked-model";
2068 let key = format!("{agent}:{model}");
2069
2070 for message in [
2072 "This model is not available in your country.",
2073 "This model is not available in your region.",
2074 ] {
2075 cooldown::reset_global_state().await;
2076 let err = AgentError::ModelUnavailable {
2077 model: model.to_string(),
2078 message: message.to_string(),
2079 };
2080
2081 let _result = handle_error(
2082 "test-3650",
2083 &err,
2084 agent,
2085 &runner,
2086 Some(model),
2087 Some("medium"),
2088 1,
2089 &None,
2090 "owner/repo",
2091 )
2092 .await
2093 .unwrap();
2094
2095 let now = chrono::Utc::now().timestamp();
2096 let until =
2097 cooldown::cooldown_until(&key).expect("region block should set model cooldown");
2098 let remaining = until.saturating_sub(now);
2099
2100 assert!(
2101 remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2102 "expected ~4h (persistent) cooldown for '{message}', got {remaining}s — \
2103 may have applied transient 5min cooldown instead (issue #3650)"
2104 );
2105 }
2106 }
2107
2108 #[serial(cooldown_state)]
2114 #[tokio::test]
2115 async fn thinking_block_conflict_does_not_set_cooldown() {
2116 cooldown::reset_global_state().await;
2117 let runner = MockRunner { free: vec![] };
2118 let agent = "claude-3204-thinking-block";
2119 let model = "opus";
2120
2121 assert!(
2122 !cooldown::is_agent_in_cooldown(agent),
2123 "agent must not be in cooldown before handle_error"
2124 );
2125 assert!(
2126 !cooldown::is_model_in_cooldown(agent, model),
2127 "model must not be in cooldown before handle_error"
2128 );
2129
2130 let err = AgentError::ThinkingBlockConflict {
2131 message: "`thinking` or `redacted_thinking` blocks in the latest assistant message cannot be modified.".to_string(),
2132 };
2133
2134 let _result = handle_error(
2135 "test-3204-thinking",
2136 &err,
2137 agent,
2138 &runner,
2139 Some(model),
2140 Some("complex"),
2141 1,
2142 &None,
2143 "owner/repo",
2144 )
2145 .await
2146 .unwrap();
2147
2148 assert!(
2149 !cooldown::is_agent_in_cooldown(agent),
2150 "agent must NOT be in cooldown after ThinkingBlockConflict (fixes issue #3204)"
2151 );
2152 assert!(
2153 !cooldown::is_model_in_cooldown(agent, model),
2154 "model must NOT be in cooldown after ThinkingBlockConflict (fixes issue #3204)"
2155 );
2156 }
2157
2158 #[serial(cooldown_state)]
2160 #[tokio::test]
2161 async fn model_unavailable_transient_applies_short_cooldown() {
2162 cooldown::reset_global_state().await;
2163 let runner = MockRunner { free: vec![] };
2164 let agent = "opencode-2941-transient";
2165 let model = "github-copilot/gpt-4o";
2166 let key = format!("{agent}:{model}");
2167
2168 let err = AgentError::ModelUnavailable {
2169 model: model.to_string(),
2170 message: "No endpoints found for github-copilot/gpt-4o.".to_string(),
2171 };
2172
2173 let _result = handle_error(
2174 "test-2941-b",
2175 &err,
2176 agent,
2177 &runner,
2178 Some(model),
2179 Some("medium"),
2180 1,
2181 &None,
2182 "owner/repo",
2183 )
2184 .await
2185 .unwrap();
2186
2187 let now = chrono::Utc::now().timestamp();
2188 let until = cooldown::cooldown_until(&key)
2189 .expect("transient model unavailability should set model cooldown");
2190 let remaining = until.saturating_sub(now);
2191
2192 assert!(
2195 remaining < cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2196 "transient unavailability should use short cooldown (<4h), got {remaining}s"
2197 );
2198 }
2199
2200 #[serial(cooldown_state)]
2201 #[tokio::test]
2202 async fn model_unavailable_does_not_reuse_same_agent_when_agent_cooled() {
2203 cooldown::reset_global_state().await;
2204 let agent = "codex-3620";
2205 let failing_model = "gpt-5.4";
2206 let other_model = "gpt-5.5";
2207 let runner = MockRunnerWithModels {
2208 models: vec![failing_model.to_string(), other_model.to_string()],
2209 };
2210
2211 cooldown::set_agent_cooldown(agent, 3600).await;
2212
2213 let err = AgentError::ModelUnavailable {
2214 model: failing_model.to_string(),
2215 message: "model gpt-5.4 unavailable".to_string(),
2216 };
2217
2218 let result = handle_error(
2219 "test-3620",
2220 &err,
2221 agent,
2222 &runner,
2223 Some(failing_model),
2224 Some("medium"),
2225 1,
2226 &None,
2227 "owner/repo",
2228 )
2229 .await
2230 .unwrap();
2231
2232 if let ErrorHandleResult::EarlyReturn { status } = result {
2233 assert_ne!(
2234 status, "routed",
2235 "agent-wide cooldown was defeated by same-agent model substitution"
2236 );
2237 }
2238 }
2239}