1use crate::store;
8use crate::store::TaskStore;
9use std::collections::HashSet;
10use std::sync::Arc;
11
12use super::{agents, response};
13use crate::cmd_cache;
14use crate::engine;
15use crate::engine::cooldown;
16use crate::engine::cooldown::CreditExhaustionReason;
17
18pub(crate) fn summarize_rate_limit_error(raw_output: &str) -> String {
29 const FIRST_LINE_CAP: usize = 120;
31 let last_api_retry_line = raw_output
35 .lines()
36 .rev()
37 .find(|line| line.contains("\"subtype\":\"api_retry\""));
38
39 if let Some(line) = last_api_retry_line {
40 return summarise_api_retry_line(line);
41 }
42
43 let api_retry_field_markers = ["\"error_status\":", "\"attempt\":", "\"retry_delay_ms\":"];
49 let field_line = raw_output
50 .lines()
51 .rev()
52 .find(|line| api_retry_field_markers.iter().any(|m| line.contains(m)));
53 if let Some(line) = field_line {
54 return summarise_api_retry_line(line);
55 }
56
57 let first_line = raw_output.lines().next().unwrap_or(raw_output);
61 if first_line.len() > FIRST_LINE_CAP {
62 let cutoff = agents::truncate_at_char_boundary(first_line, FIRST_LINE_CAP);
63 let remaining = raw_output.len().saturating_sub(cutoff);
64 format!("{}... (+{} more chars)", &first_line[..cutoff], remaining)
65 } else {
66 first_line.to_string()
67 }
68}
69
70fn summarise_api_retry_line(line: &str) -> String {
75 let error_status =
76 extract_json_field(line, "\"error_status\":").unwrap_or_else(|| "unknown".to_string());
77 let attempt = extract_json_field(line, "\"attempt\":").unwrap_or_else(|| "unknown".to_string());
78 let retry_delay_ms =
79 extract_json_field(line, "\"retry_delay_ms\":").unwrap_or_else(|| "0".to_string());
80
81 let has_field = error_status != "unknown"
82 || attempt != "unknown"
83 || (retry_delay_ms != "0" && retry_delay_ms != "unknown");
84
85 if has_field {
86 let retry_delay_s = retry_delay_ms.parse::<f64>().unwrap_or(0.0) / 1000.0;
87 #[expect(
88 clippy::cast_possible_truncation,
89 clippy::cast_sign_loss,
90 reason = "the saturating cast floors the delay to whole seconds"
91 )]
92 let retry_delay_s = retry_delay_s as u64;
93 format!("status={error_status} after {attempt} attempts (last delay {retry_delay_s}s)")
94 } else {
95 let status_hint = extract_json_field(line, "\"status\":")
96 .or_else(|| extract_json_field(line, "\"error\":"))
97 .unwrap_or_else(|| "429".to_string());
98 format!("status={status_hint} (api_retry fragment)")
99 }
100}
101
102fn extract_json_field(json: &str, field: &str) -> Option<String> {
104 json.find(field).and_then(|start| {
105 let value_start = start + field.len();
106 let value_str = &json[value_start..];
107
108 if let Some(rest) = value_str.strip_prefix('"') {
110 let end = rest.find('"')?; Some(rest[..end].to_string())
112 }
113 else {
115 let end = value_str
116 .find(|c: char| !c.is_ascii_digit() && c != '.')
117 .unwrap_or(value_str.len());
118 let num_str = &value_str[..end];
119 num_str
120 .strip_suffix(',')
121 .map(ToString::to_string)
122 .or_else(|| Some(num_str.to_string()))
123 }
124 })
125}
126
127#[derive(Debug)]
129pub enum ErrorHandleResult {
130 EarlyReturn { status: String },
132 Continue { status: String, error: String },
138}
139
140#[expect(
151 clippy::too_many_arguments,
152 reason = "each caller picks a different mix of reroute options"
153)]
154#[expect(
155 clippy::ref_option,
156 reason = "the store helpers it calls take the store as &Option"
157)]
158async fn try_free_model_reroute(
159 task_id: &str,
160 agent_name: &str,
161 agent_runner: &dyn agents::AgentRunner,
162 exclude_models: &[&str],
163 check_cooldowns: bool,
164 set_agent: Option<&str>,
165 apply_backoff: bool,
166 reason_prefix: &str,
167 store: &Option<Arc<TaskStore>>,
168 repo: &str,
169) -> Option<ErrorHandleResult> {
170 let free = agent_runner.free_models();
171 if free.is_empty() {
172 return None;
173 }
174 let tried_models: String = store::opt_store_get_task(store, repo, task_id)
175 .await
176 .map(|t| t.model_reroute_chain)
177 .unwrap_or_default();
178 let tried_set: HashSet<&str> = tried_models.split(',').filter(|s| !s.is_empty()).collect();
179 let next_free = free.iter().find(|m| {
180 !exclude_models.contains(&m.as_str())
181 && !tried_set.contains(m.as_str())
182 && (!check_cooldowns
183 || (!cooldown::is_agent_in_cooldown(agent_name)
184 && !response::is_model_in_cooldown(agent_name, m)))
185 })?;
186 tracing::info!(task_id, model = %next_free, "{}", reason_prefix);
187 let new_tried = if tried_models.is_empty() {
188 next_free.clone()
189 } else {
190 format!("{tried_models},{next_free}")
191 };
192 let msg = format!("{reason_prefix} {next_free}");
193 let mut fields: Vec<(&str, serde_json::Value)> = vec![
194 ("model", serde_json::json!(next_free.clone())),
195 ("model_reroute_chain", serde_json::json!(new_tried)),
196 ("last_error", serde_json::json!(msg)),
197 ];
198 if let Some(agent) = set_agent {
199 fields.push(("agent", serde_json::json!(agent)));
200 }
201 if let Err(e) = store::store_set_result(store, repo, task_id, &fields).await {
202 tracing::warn!(task_id, err = %e, "try_free_model_reroute: failed to persist model update — not rerouting");
203 return None; }
205 if apply_backoff {
206 response::wait_for_fallback_backoff(task_id, store, repo).await;
207 }
208 Some(ErrorHandleResult::EarlyReturn {
209 status: "new".to_string(),
210 })
211}
212
213async fn apply_vendor_reset_date(
218 agent_name: &str,
219 model_name: Option<&str>,
220 message: &str,
221) -> bool {
222 let Some(model) = model_name else {
223 return false;
224 };
225 match cooldown::detect_credit_exhaustion(message) {
226 None | Some(CreditExhaustionReason::BillingCycleExhausted) => {}
227 Some(_) => return false,
228 }
229 match cooldown::parse_retry_at(message) {
230 Some(until) if until > chrono::Utc::now().timestamp() => {
231 cooldown::set_model_cooldown_until(agent_name, model, until).await;
232 true
233 }
234 _ => false,
235 }
236}
237
238#[expect(
244 clippy::too_many_arguments,
245 reason = "the error path needs the task, agent, model and store context together"
246)]
247pub async fn handle_error(
248 task_id: &str,
249 agent_err: &agents::AgentError,
250 agent_name: &str,
251 agent_runner: &dyn agents::AgentRunner,
252 model_name: Option<&str>,
253 complexity: Option<&str>,
254 new_attempts: u32,
255 store: &Option<Arc<TaskStore>>,
256 repo: &str,
257) -> anyhow::Result<ErrorHandleResult> {
258 tracing::warn!(
259 task_id,
260 agent = agent_name,
261 error = %agent_err,
262 "agent error, attempting recovery"
263 );
264
265 let mut credit_recorded = false;
268 let (retryable, error_msg) = match agent_err {
269 agents::AgentError::RateLimit { message, .. } => {
270 let is_provider_returned_error = message
271 .to_ascii_lowercase()
272 .contains("provider returned error");
273 let mut billing_cycle_model_scoped = false;
293
294 if apply_vendor_reset_date(agent_name, model_name, message).await {
295 let reason = cooldown::detect_credit_exhaustion(message);
296 credit_recorded = reason.is_some();
297 billing_cycle_model_scoped =
298 reason == Some(CreditExhaustionReason::BillingCycleExhausted);
299 } else if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
300 credit_recorded = true;
301 if reason == CreditExhaustionReason::BillingCycleExhausted {
307 if let Some(model) = model_name {
308 cooldown::record_persistent_model_failure(agent_name, model).await;
309 billing_cycle_model_scoped = true;
310 } else {
311 cooldown::record_credit_exhaustion(agent_name, reason).await;
312 }
313 } else {
314 cooldown::record_credit_exhaustion(agent_name, reason).await;
315 }
316 } else if let (Some(model), Some(window_secs)) =
317 (model_name, cooldown::parse_relative_usage_window(message))
318 {
319 cooldown::set_model_cooldown(
327 agent_name,
328 model,
329 window_secs,
330 "rate_limit_usage_window",
331 )
332 .await;
333 tracing::info!(
334 task_id,
335 agent = agent_name,
336 model,
337 window_secs,
338 "relative usage window detected: applying model-specific cooldown"
339 );
340 } else {
341 cooldown::record_agent_failure_with_message(agent_name, message).await;
342 if let Some(model) = model_name {
344 if is_provider_returned_error {
345 cooldown::record_transient_model_failure(agent_name, model).await;
346 } else {
347 response::record_model_failure(agent_name, model).await;
348 }
349 }
350 }
351 let retryable = if billing_cycle_model_scoped {
355 response::RetryableError::ModelUnavailable
356 } else {
357 response::RetryableError::UsageLimit
358 };
359 (
360 retryable,
361 format!(
362 "{agent_name} rate limit: {}",
363 summarize_rate_limit_error(message)
364 ),
365 )
366 }
367 agents::AgentError::Auth { message } => {
368 let mut billing_cycle_model_scoped = false;
373
374 if apply_vendor_reset_date(agent_name, model_name, message).await {
375 let reason = cooldown::detect_credit_exhaustion(message);
376 credit_recorded = reason.is_some();
377 billing_cycle_model_scoped =
378 reason == Some(CreditExhaustionReason::BillingCycleExhausted);
379 } else if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
380 credit_recorded = true;
381 if reason == CreditExhaustionReason::BillingCycleExhausted {
384 if let Some(model) = model_name {
385 cooldown::record_persistent_model_failure(agent_name, model).await;
386 billing_cycle_model_scoped = true;
387 } else {
388 cooldown::record_credit_exhaustion(agent_name, reason).await;
389 }
390 } else {
391 cooldown::record_credit_exhaustion(agent_name, reason).await;
392 }
393 } else {
394 cooldown::record_agent_failure_with_message(agent_name, message).await;
395 if let Some(model) = model_name {
396 response::record_model_failure(agent_name, model).await;
397 }
398 }
399 let retryable = if billing_cycle_model_scoped {
402 response::RetryableError::ModelUnavailable
403 } else {
404 response::RetryableError::AuthError
405 };
406 (retryable, format!("{agent_name} auth error: {message}"))
407 }
408 agents::AgentError::Timeout { elapsed } => {
409 (
412 response::RetryableError::Timeout,
413 format!("{agent_name} timed out after {}s", elapsed.as_secs()),
414 )
415 }
416 agents::AgentError::Interrupted { message } => (
417 response::RetryableError::Interrupted,
418 format!("{agent_name} interrupted: {message}"),
419 ),
420 agents::AgentError::MissingTool { tool } => (
421 response::RetryableError::MissingTooling,
422 format!("missing tool: {tool}"),
423 ),
424 agents::AgentError::ModelUnavailable { model, message } => {
425 cooldown::record_model_unavailable(agent_name, model, message).await;
426
427 let agent_cooled = cooldown::is_agent_in_cooldown(agent_name);
430 let models = agent_runner.available_models();
431 let current_model = model_name.unwrap_or("");
432 let next_model = if agent_cooled {
433 None
434 } else {
435 models.iter().find(|m| {
436 m.as_str() != current_model
437 && m.as_str() != model
438 && !response::is_model_in_cooldown(agent_name, m)
439 })
440 };
441 if let Some(next) = next_model {
442 tracing::info!(task_id, model = %next, "retrying with different model");
443 let msg = format!("model {model} unavailable, trying {next}");
444 if let Err(e) = store::store_set_result(
445 store,
446 repo,
447 task_id,
448 &[
449 ("model", serde_json::json!(next.clone())),
450 ("last_error", serde_json::json!(msg)),
451 ],
452 )
453 .await
454 {
455 tracing::warn!(task_id, err = %e, "failed to write model failover to store");
456 }
457 response::wait_for_fallback_backoff(task_id, store, repo).await;
461 return Ok(ErrorHandleResult::EarlyReturn {
462 status: "routed".to_string(),
463 });
464 }
465
466 let is_simple = matches!(complexity, None | Some("simple"));
468 if is_simple
469 && let Some(result) = try_free_model_reroute(
470 task_id,
471 agent_name,
472 agent_runner,
473 &[current_model, model.as_str()],
474 true,
475 None,
476 true,
477 &format!("model {model} unavailable, trying free model"),
478 store,
479 repo,
480 )
481 .await
482 {
483 return Ok(result);
484 }
485
486 (
487 response::RetryableError::ModelUnavailable,
488 format!("model {model} unavailable"),
489 )
490 }
491 agents::AgentError::ContextOverflow { message } => {
492 let msg = format!("{agent_name} context overflow: {message}");
498 if let Err(e) = store::store_set_result(
499 store,
500 repo,
501 task_id,
502 &[("last_error", serde_json::json!(msg))],
503 )
504 .await
505 {
506 tracing::warn!(task_id, err = %e, "failed to write context_overflow last_error to store");
507 }
508 return Ok(ErrorHandleResult::EarlyReturn {
509 status: "needs_review".to_string(),
510 });
511 }
512 agents::AgentError::WaitingForInput { message } => {
513 let msg = format!("waiting for input: {message}");
515 if let Err(e) = store::store_set_result(
516 store,
517 repo,
518 task_id,
519 &[("last_error", serde_json::json!(msg))],
520 )
521 .await
522 {
523 tracing::warn!(task_id, err = %e, "failed to write waiting_for_input last_error to store");
524 }
525 return Ok(ErrorHandleResult::EarlyReturn {
526 status: "needs_review".to_string(),
527 });
528 }
529 agents::AgentError::PermissionDenied { message } => (
530 response::RetryableError::Failed,
531 format!("permission denied: {message}"),
532 ),
533 agents::AgentError::InvalidResponse { .. } => {
534 if let Some(model) = model_name {
537 response::record_model_failure(agent_name, model).await;
538 }
539 (
540 response::RetryableError::Failed,
541 format!("{agent_name} invalid response"),
542 )
543 }
544 agents::AgentError::AgentFailed { message } => {
545 if let Some(reason) = cooldown::detect_credit_exhaustion(message) {
552 credit_recorded = true;
553 match model_name {
554 Some(model) if reason == CreditExhaustionReason::BillingCycleExhausted => {
555 cooldown::record_persistent_model_failure(agent_name, model).await;
556 }
557 _ => cooldown::record_credit_exhaustion(agent_name, reason).await,
558 }
559 } else {
560 if let Some(model) = model_name {
561 response::record_model_failure(agent_name, model).await;
562 }
563 cooldown::record_agent_failure_with_message(agent_name, message).await;
564 }
565 (
566 response::RetryableError::Failed,
567 format!("{agent_name} failed: {message}"),
568 )
569 }
570 agents::AgentError::NetworkError { message } => {
571 const MAX_NETWORK_RETRIES: u64 = 8;
577 let msg = format!("{agent_name} network error: {message}");
578 let retry_count = match store::store_increment(store, repo, task_id, "network_retries")
579 .await
580 {
581 Ok(count) => {
582 tracing::info!(
583 task_id,
584 retry_count = count,
585 agent = agent_name,
586 "network retry scheduled"
587 );
588 count
589 }
590 Err(e) => {
591 tracing::warn!(
592 task_id,
593 error = %e,
594 "failed to increment network_retries — escalating to needs_review"
595 );
596 return Ok(ErrorHandleResult::Continue {
597 status: "needs_review".to_string(),
598 error: format!("{agent_name} network error (store unavailable): {message}"),
599 });
600 }
601 };
602 if let Err(e) = store::store_set_result(
603 store,
604 repo,
605 task_id,
606 &[("last_error", serde_json::json!(msg))],
607 )
608 .await
609 {
610 tracing::warn!(task_id, err = %e, "failed to write network_error last_error to store");
611 }
612 if retry_count >= MAX_NETWORK_RETRIES {
613 tracing::warn!(
614 task_id,
615 retry_count,
616 agent = agent_name,
617 "network retries exhausted, escalating to needs_review"
618 );
619 return Ok(ErrorHandleResult::Continue {
620 status: "needs_review".to_string(),
621 error: msg,
622 });
623 }
624 response::wait_for_fallback_backoff(task_id, store, repo).await;
626 return Ok(ErrorHandleResult::EarlyReturn {
627 status: "routed".to_string(),
628 });
629 }
630 agents::AgentError::StaleSession { session_id } => {
631 (
635 response::RetryableError::Failed,
636 format!("{agent_name} stale session: {session_id}"),
637 )
638 }
639 agents::AgentError::ThinkingBlockConflict { message } => {
640 (
646 response::RetryableError::ModelUnavailable,
647 format!("{agent_name} thinking block conflict: {message}"),
648 )
649 }
650 agents::AgentError::Unknown { exit_code, message } => {
651 if *exit_code == 0
657 && (message.is_empty()
658 || message == "empty-output-exit0"
659 || message.starts_with("empty-output-exit0:"))
660 {
661 if let Some(m) = model_name {
662 cooldown::record_transient_model_failure(agent_name, m).await;
664 }
665 let current = model_name.unwrap_or("");
670 if let Some(result) = try_free_model_reroute(
671 task_id,
672 agent_name,
673 agent_runner,
674 &[current],
675 true,
676 Some("opencode"),
677 true,
678 "silent exit 0, retrying with free model",
679 store,
680 repo,
681 )
682 .await
683 {
684 return Ok(result);
685 }
686 }
687
688 let error_message = if *exit_code != 0 && message.contains("\"costUSD\":") {
691 format!(
693 "{agent_name} exited with non-zero status (cost telemetry in stdout, no error message)"
694 )
695 } else {
696 format!("{agent_name} exit {exit_code}: {message}")
697 };
698
699 (response::RetryableError::Failed, error_message)
700 }
701 };
702
703 if matches!(
705 retryable,
706 response::RetryableError::Timeout | response::RetryableError::Interrupted
707 ) && let Err(e) = store::store_set_result(
708 store,
709 repo,
710 task_id,
711 &[
712 ("agent", serde_json::json!("")),
713 ("last_error", serde_json::json!(error_msg.clone())),
714 ],
715 )
716 .await
717 {
718 tracing::warn!(task_id, err = %e, "failed to write timeout agent clear to store");
719 }
720
721 {
724 let error_type_str = match retryable {
725 response::RetryableError::UsageLimit => Some("rate"),
726 response::RetryableError::AuthError => Some("auth"),
727 response::RetryableError::Timeout | response::RetryableError::Interrupted => None,
728 _ => Some("error"),
729 };
730 if let Some(limit_type) = error_type_str
731 && let Some(s) = store
732 && let Err(e) = s
733 .record_rate_limit(agent_name, limit_type, Some(task_id))
734 .await
735 {
736 tracing::warn!(task_id, agent = %agent_name, err = %e, "failed to persist rate_limit event — router health check may miss this signal");
737 }
738 }
739
740 let chain = response::get_reroute_chain(task_id, store, repo).await;
742 let available: Vec<String> = engine::configured_agents()
743 .into_iter()
744 .filter(|a| cmd_cache::command_exists(a))
745 .collect();
746
747 let all_agents_tried = {
748 let chain_set: HashSet<&str> = if chain.is_empty() {
749 HashSet::new()
750 } else {
751 chain.split(',').collect()
752 };
753 !available
754 .iter()
755 .any(|a| a.as_str() != agent_name && !chain_set.contains(a.as_str()))
756 };
757
758 let is_simple = matches!(complexity, None | Some("simple"));
759 if all_agents_tried && is_simple {
760 if let Some(result) = try_free_model_reroute(
762 task_id,
763 agent_name,
764 agent_runner,
765 &[],
766 true, Some("opencode"),
768 false,
769 "all agents exhausted, trying free model",
770 store,
771 repo,
772 )
773 .await
774 {
775 return Ok(result);
776 }
777 }
778
779 let outcome = response::handle_failover(
780 task_id,
781 agent_name,
782 retryable,
783 &error_msg,
784 credit_recorded,
785 store,
786 repo,
787 )
788 .await;
789 match (outcome.status.as_str(), outcome.next_agent.as_deref()) {
790 ("new", Some(next)) => {
791 tracing::info!(task_id, to = next, "failover: re-routing to fallback agent");
792 }
793 ("new", None) => {
794 tracing::warn!(task_id, "failover exhausted (retryable), task reset to new");
795 }
796 ("blocked", _) => {
797 tracing::warn!(task_id, "failover exhausted (hard failure), task blocked");
798 }
799 _ => {}
800 }
801 let status = outcome.status;
802
803 response::store_failure_memory(
805 task_id,
806 new_attempts,
807 agent_name,
808 model_name,
809 &error_msg,
810 store,
811 repo,
812 )
813 .await;
814
815 Ok(ErrorHandleResult::Continue {
816 status,
817 error: error_msg,
818 })
819}
820
821#[cfg(test)]
822mod tests {
823 use std::time::Duration;
824
825 use super::*;
826 use crate::engine::runner::agents::StreamEvent;
827 use crate::engine::runner::agents::{AgentError, AgentRunner, ParsedResponse, PermissionRules};
828 use crate::parser::AgentResponse;
829 use chrono::Datelike;
830 use serial_test::serial;
831 use tokio::process::Command;
832
833 struct MockRunner {
834 free: Vec<String>,
835 }
836
837 struct MockRunnerWithModels {
838 models: Vec<String>,
839 }
840
841 impl AgentRunner for MockRunnerWithModels {
842 fn stream_events(&self, _event: &serde_json::Value) -> Vec<StreamEvent> {
843 vec![]
844 }
845
846 fn name(&self) -> &'static str {
847 "opencode"
848 }
849
850 fn build_command(
851 &self,
852 _model: Option<&str>,
853 _timeout_cmd: &str,
854 _sys_file: &str,
855 _msg_file: &str,
856 _permissions: &PermissionRules,
857 ) -> String {
858 String::new()
859 }
860
861 fn parse_response(&self, _raw: &str) -> Result<ParsedResponse, AgentError> {
862 Ok(ParsedResponse {
863 response: AgentResponse {
864 status: "done".to_string(),
865 summary: String::new(),
866 accomplished: vec![],
867 remaining: vec![],
868 files: vec![],
869 error: None,
870 input_tokens: None,
871 output_tokens: None,
872 learnings: vec![],
873 delegations: vec![],
874 },
875 input_tokens: None,
876 output_tokens: None,
877 cost_usd: None,
878 duration_ms: None,
879 })
880 }
881
882 fn classify_error(&self, _exit_code: i32, _stdout: &str, _stderr: &str) -> AgentError {
883 AgentError::Unknown {
884 exit_code: 0,
885 message: String::new(),
886 }
887 }
888
889 fn available_models(&self) -> Vec<String> {
890 self.models.clone()
891 }
892
893 fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
894 anyhow::bail!("not implemented")
895 }
896 }
897
898 impl AgentRunner for MockRunner {
899 fn stream_events(&self, _event: &serde_json::Value) -> Vec<StreamEvent> {
900 vec![]
901 }
902
903 fn name(&self) -> &'static str {
904 "opencode"
905 }
906
907 fn build_command(
908 &self,
909 _model: Option<&str>,
910 _timeout_cmd: &str,
911 _sys_file: &str,
912 _msg_file: &str,
913 _permissions: &PermissionRules,
914 ) -> String {
915 String::new()
916 }
917
918 fn parse_response(&self, _raw: &str) -> Result<ParsedResponse, AgentError> {
919 Ok(ParsedResponse {
920 response: AgentResponse {
921 status: "done".to_string(),
922 summary: String::new(),
923 accomplished: vec![],
924 remaining: vec![],
925 files: vec![],
926 error: None,
927 input_tokens: None,
928 output_tokens: None,
929 learnings: vec![],
930 delegations: vec![],
931 },
932 input_tokens: None,
933 output_tokens: None,
934 cost_usd: None,
935 duration_ms: None,
936 })
937 }
938
939 fn classify_error(&self, _exit_code: i32, _stdout: &str, _stderr: &str) -> AgentError {
940 AgentError::Unknown {
941 exit_code: 0,
942 message: String::new(),
943 }
944 }
945
946 fn free_models(&self) -> Vec<String> {
947 self.free.clone()
948 }
949
950 fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
951 anyhow::bail!("not implemented")
952 }
953 }
954
955 #[serial(cooldown_state)]
961 #[tokio::test]
962 async fn rate_limit_sets_agent_cooldown_immediately() {
963 cooldown::reset_global_state().await;
964 let runner = MockRunner { free: vec![] };
965 let agent = "test-agent-1371-early-cooldown";
966
967 assert!(
969 !cooldown::is_agent_in_cooldown(agent),
970 "agent should not be in cooldown before handle_error"
971 );
972
973 let err = AgentError::RateLimit {
974 message: "429 Too Many Requests".to_string(),
975 };
976
977 let _result = handle_error(
979 "test-1371-a",
980 &err,
981 agent,
982 &runner,
983 Some("sonnet"),
984 Some("medium"),
985 1,
986 &None,
987 "owner/repo",
988 )
989 .await
990 .unwrap();
991
992 assert!(
994 cooldown::is_agent_in_cooldown(agent),
995 "agent should be in cooldown immediately after handle_error for RateLimit"
996 );
997 }
998
999 #[serial(cooldown_state)]
1006 #[tokio::test]
1007 async fn rate_limit_sets_both_agent_and_model_cooldowns() {
1008 cooldown::reset_global_state().await;
1009 let runner = MockRunner { free: vec![] };
1010 let agent = "test-agent-2153-model-cooldown";
1011 let model = "opencode/github-copilot/qwen3.6-plus-free";
1012
1013 assert!(
1015 !cooldown::is_agent_in_cooldown(agent),
1016 "agent should not be in cooldown before handle_error"
1017 );
1018 assert!(
1019 !cooldown::is_model_in_cooldown(agent, model),
1020 "model should not be in cooldown before handle_error"
1021 );
1022
1023 let err = AgentError::RateLimit {
1024 message: "Upstream error from Alibaba: Request rate increased too quickly.".to_string(),
1025 };
1026
1027 let _result = handle_error(
1028 "test-2153-a",
1029 &err,
1030 agent,
1031 &runner,
1032 Some(model),
1033 Some("simple"),
1034 1,
1035 &None,
1036 "owner/repo",
1037 )
1038 .await
1039 .unwrap();
1040
1041 assert!(
1043 cooldown::is_agent_in_cooldown(agent),
1044 "agent should be in agent-level cooldown after RateLimit"
1045 );
1046 assert!(
1049 cooldown::is_model_in_cooldown(agent, model),
1050 "model should be in model-level cooldown after RateLimit (fixes issue #2153)"
1051 );
1052 }
1053
1054 #[serial(cooldown_state)]
1056 #[tokio::test]
1057 async fn rate_limit_cooldown_is_model_specific() {
1058 cooldown::reset_global_state().await;
1059 let runner = MockRunner { free: vec![] };
1060 let agent = "test-agent-2153-model-specific";
1061 let cooled_model = "opencode/github-copilot/qwen3.6-plus-free";
1062 let other_model = "opencode/minimax-m2.5-free";
1063
1064 let err = AgentError::RateLimit {
1065 message: "rate limit exceeded".to_string(),
1066 };
1067
1068 let _result = handle_error(
1069 "test-2153-b",
1070 &err,
1071 agent,
1072 &runner,
1073 Some(cooled_model),
1074 Some("simple"),
1075 1,
1076 &None,
1077 "owner/repo",
1078 )
1079 .await
1080 .unwrap();
1081
1082 assert!(
1084 cooldown::is_model_in_cooldown(agent, cooled_model),
1085 "rate-limited model should be in cooldown"
1086 );
1087 assert!(
1089 !cooldown::is_model_in_cooldown(agent, other_model),
1090 "other models should not be cooled by a single model's rate limit"
1091 );
1092 }
1093
1094 #[serial(cooldown_state)]
1105 #[tokio::test]
1106 async fn credit_exhaustion_does_not_bump_generic_failure_count() {
1107 cooldown::reset_global_state().await;
1108 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1109 cooldown::init_cooldown_store(Arc::clone(&store)).await;
1110
1111 let runner = MockRunner { free: vec![] };
1112 let agent = "test-agent-3652-credit-no-bump";
1113
1114 let err = AgentError::RateLimit {
1115 message: "out of credits, please recharge your balance".to_string(),
1116 };
1117
1118 let _result = handle_error(
1119 "test-3652-a",
1120 &err,
1121 agent,
1122 &runner,
1123 None,
1124 Some("simple"),
1125 1,
1126 &Some(Arc::clone(&store)),
1127 "owner/repo",
1128 )
1129 .await
1130 .unwrap();
1131
1132 let credit_count = store
1133 .kv_get(&format!("credit_failure_count:{agent}"))
1134 .await
1135 .unwrap();
1136 assert!(
1137 credit_count.is_some(),
1138 "credit_failure_count should be incremented by the credit error"
1139 );
1140 let generic_count = store
1141 .kv_get(&format!("failure_count:{agent}"))
1142 .await
1143 .unwrap();
1144 assert!(
1145 generic_count.is_none(),
1146 "failure_count must not be bumped by credit errors, got {generic_count:?}"
1147 );
1148 }
1149
1150 #[serial(cooldown_state)]
1153 #[tokio::test]
1154 async fn two_timeouts_leave_model_routable() {
1155 cooldown::reset_global_state().await;
1156 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1157 cooldown::init_cooldown_store(Arc::clone(&store)).await;
1158
1159 let runner = MockRunner { free: vec![] };
1160 let agent = "opencode";
1161 let model = "opencode/nemotron-3-ultra-free";
1162
1163 let err = AgentError::Timeout {
1164 elapsed: Duration::from_mins(30),
1165 };
1166
1167 for i in 1..=2 {
1168 let _result = handle_error(
1169 &format!("test-3876-timeout-{i}"),
1170 &err,
1171 agent,
1172 &runner,
1173 Some(model),
1174 Some("simple"),
1175 i,
1176 &Some(Arc::clone(&store)),
1177 "owner/repo",
1178 )
1179 .await
1180 .unwrap();
1181 }
1182
1183 assert!(
1184 !cooldown::is_model_in_cooldown(agent, model),
1185 "timeouts must not cool the model"
1186 );
1187 assert!(
1188 !cooldown::is_agent_in_cooldown(agent),
1189 "timeouts must not cool the agent"
1190 );
1191 let counts = store.recent_rate_limit_counts(6).await.unwrap();
1192 assert_eq!(
1193 counts.get(agent).copied().unwrap_or(0),
1194 0,
1195 "timeouts must not write a rate_limits row"
1196 );
1197 }
1198
1199 #[serial(cooldown_state)]
1202 #[tokio::test]
1203 async fn interrupted_run_does_not_write_rate_limit() {
1204 cooldown::reset_global_state().await;
1205 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
1206 cooldown::init_cooldown_store(Arc::clone(&store)).await;
1207
1208 let runner = MockRunner { free: vec![] };
1209 let agent = "claude";
1210 let model = "sonnet";
1211 let err = AgentError::Interrupted {
1212 message: "exited without a result envelope".to_string(),
1213 };
1214
1215 let _result = handle_error(
1216 "test-3876-interrupted",
1217 &err,
1218 agent,
1219 &runner,
1220 Some(model),
1221 Some("medium"),
1222 1,
1223 &Some(Arc::clone(&store)),
1224 "owner/repo",
1225 )
1226 .await
1227 .unwrap();
1228
1229 assert!(
1230 !cooldown::is_model_in_cooldown(agent, model),
1231 "interrupted runs must not cool the model"
1232 );
1233 let counts = store.recent_rate_limit_counts(6).await.unwrap();
1234 assert_eq!(
1235 counts.get(agent).copied().unwrap_or(0),
1236 0,
1237 "interrupted runs must not write a rate_limits row"
1238 );
1239 }
1240
1241 #[serial(cooldown_state)]
1254 #[tokio::test]
1255 async fn rate_limit_with_relative_window_sets_window_length_model_cooldown() {
1256 cooldown::reset_global_state().await;
1257 let runner = MockRunner { free: vec![] };
1258 let agent = "test-agent-3580-kimi";
1259 let model = "opus";
1260 let key = format!("{agent}:{model}");
1261
1262 let err = AgentError::RateLimit {
1263 message: "You've reached your weekly (7-day) usage limit. Your quota will reset when the current 7-day window expires.".to_string(),
1264 };
1265
1266 let _result = handle_error(
1267 "test-3580-a",
1268 &err,
1269 agent,
1270 &runner,
1271 Some(model),
1272 Some("medium"),
1273 1,
1274 &None,
1275 "owner/repo",
1276 )
1277 .await
1278 .unwrap();
1279
1280 let now = chrono::Utc::now().timestamp();
1281 let until = cooldown::cooldown_until(&key)
1282 .expect("relative usage window should set model cooldown");
1283 let remaining = until.saturating_sub(now);
1284
1285 assert!(
1286 remaining >= 7 * 86400 - 30,
1287 "expected ~7-day model cooldown from the parsed window, got {remaining}s"
1288 );
1289 assert_eq!(
1290 cooldown::cooldown_reason(&key).as_deref(),
1291 Some("rate_limit_usage_window"),
1292 "relative usage-window cooldown must be labeled as a rate limit, not silence"
1293 );
1294 }
1295
1296 #[serial(cooldown_state)]
1297 #[tokio::test]
1298 async fn provider_returned_error_sets_short_model_cooldown() {
1299 cooldown::reset_global_state().await;
1300 let runner = MockRunner { free: vec![] };
1301 let agent = "opencode";
1302 let model = "opencode/nemotron-3-super-free";
1303 let key = format!("{agent}:{model}");
1304
1305 let err = AgentError::RateLimit {
1306 message: "Provider returned error".to_string(),
1307 };
1308
1309 let _result = handle_error(
1310 "test-2478-provider-cooldown",
1311 &err,
1312 agent,
1313 &runner,
1314 Some(model),
1315 Some("simple"),
1316 1,
1317 &None,
1318 "owner/repo",
1319 )
1320 .await
1321 .unwrap();
1322
1323 let now = chrono::Utc::now().timestamp();
1324 let until =
1325 cooldown::cooldown_until(&key).expect("provider error should set model cooldown");
1326 let remaining = until.saturating_sub(now);
1327
1328 assert!(
1329 remaining <= cooldown::TRANSIENT_MODEL_BACKOFF_BASE_SECS,
1330 "expected a 5 minute model cooldown, got {remaining}s"
1331 );
1332 }
1333
1334 #[serial(cooldown_state)]
1335 #[tokio::test]
1336 async fn model_unavailable_sets_persistent_model_cooldown_for_codex_style() {
1337 cooldown::reset_global_state().await;
1338 let runner = MockRunner { free: vec![] };
1339 let agent = "codex";
1340 let model = "gpt-5.3-codex";
1341
1342 let err = AgentError::ModelUnavailable {
1344 model: model.to_string(),
1345 message: "{\"detail\":\"The 'gpt-5.3-codex' model is not supported\"}".to_string(),
1346 };
1347
1348 let _result = handle_error(
1349 "test-codex-model-unsupported",
1350 &err,
1351 agent,
1352 &runner,
1353 Some(model),
1354 Some("simple"),
1355 1,
1356 &None,
1357 "owner/repo",
1358 )
1359 .await
1360 .unwrap();
1361
1362 let key = format!("{agent}:{model}");
1364 let now = chrono::Utc::now().timestamp();
1365 let until = cooldown::cooldown_until(&key).expect("model cooldown should be set");
1366 let remaining = until.saturating_sub(now);
1367
1368 assert!(
1369 remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
1370 "expected persistent model cooldown (~4h) for codex model-unavailable, got {remaining}s"
1371 );
1372 }
1373
1374 async fn vendor_reset_remaining(agent: &str, model: &str, message: &str) -> i64 {
1375 cooldown::reset_global_state().await;
1376 let runner = MockRunner { free: vec![] };
1377 let err = AgentError::RateLimit {
1378 message: message.to_string(),
1379 };
1380 let _ = handle_error(
1381 "test-3649",
1382 &err,
1383 agent,
1384 &runner,
1385 Some(model),
1386 Some("simple"),
1387 1,
1388 &None,
1389 "owner/repo",
1390 )
1391 .await
1392 .unwrap();
1393 let until = cooldown::cooldown_until(&format!("{agent}:{model}"))
1394 .expect("model cooldown should be set");
1395 until - chrono::Utc::now().timestamp()
1396 }
1397
1398 #[serial(cooldown_state)]
1399 #[tokio::test]
1400 async fn vendor_reset_date_overrides_weekly_window() {
1401 let t = chrono::Local::now() + chrono::Duration::days(2);
1402 let msg = format!(
1403 "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
1404 t.format("%b"),
1405 t.day()
1406 );
1407 let remaining = vendor_reset_remaining("claude", "sonnet", &msg).await;
1408 assert!(
1409 remaining > 3600 && remaining < 3 * 86400,
1410 "expected ~2d cooldown, got {remaining}s"
1411 );
1412 }
1413
1414 #[serial(cooldown_state)]
1415 #[tokio::test]
1416 async fn vendor_reset_date_overrides_billing_cycle_backoff() {
1417 let t = chrono::Local::now() + chrono::Duration::days(3);
1418 let msg = format!(
1419 "You've hit your usage limit. Upgrade to Pro (https://x), or try again at {}.",
1420 t.format("%b %d, %Y %I:%M %p")
1421 );
1422 let remaining = vendor_reset_remaining("codex", "gpt-5.4", &msg).await;
1423 assert!(
1424 remaining > 2 * 86400 + 80000 - 3600 && remaining < 4 * 86400,
1425 "expected ~3d cooldown, got {remaining}s"
1426 );
1427 }
1428
1429 #[serial(cooldown_state)]
1430 #[tokio::test]
1431 async fn agent_failed_insufficient_balance_sets_credit_cooldown() {
1432 cooldown::reset_global_state().await;
1433 let runner = MockRunner { free: vec![] };
1434 let agent = "test-agent-3642-insufficient-balance";
1435
1436 assert!(
1438 !cooldown::is_agent_in_cooldown(agent),
1439 "agent should not be in cooldown before handle_error"
1440 );
1441
1442 let err = agents::AgentError::AgentFailed {
1443 message: "402 Payment Required: insufficient balance".to_string(),
1444 };
1445
1446 let _result = handle_error(
1448 "test-3642-a",
1449 &err,
1450 agent,
1451 &runner,
1452 Some("sonnet"),
1453 Some("medium"),
1454 1,
1455 &None,
1456 "owner/repo",
1457 )
1458 .await
1459 .unwrap();
1460
1461 assert!(
1466 cooldown::is_agent_in_cooldown(agent),
1467 "agent should be in cooldown after handle_error for AgentFailed with insufficient balance"
1468 );
1469
1470 let until = cooldown::cooldown_until(agent).expect("agent cooldown should be active");
1471 let remaining = until - chrono::Utc::now().timestamp();
1472 assert!(
1473 remaining >= cooldown::CREDIT_BACKOFF_BASE_SECS - 30,
1474 "credit cooldown should be at least the 1h base, got {remaining}s"
1475 );
1476 }
1477
1478 #[serial(cooldown_state)]
1479 #[tokio::test]
1480 async fn silent_exit0_retries_free_model_for_simple_complexity() {
1481 cooldown::reset_global_state().await;
1482 let runner = MockRunner {
1483 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1484 };
1485 let err = AgentError::Unknown {
1486 exit_code: 0,
1487 message: String::new(),
1488 };
1489
1490 let result = handle_error(
1492 "test-1112-a",
1493 &err,
1494 "opencode",
1495 &runner,
1496 Some("opencode/gpt-5.4-mini"),
1497 Some("simple"),
1498 1,
1499 &None,
1500 "owner/repo",
1501 )
1502 .await
1503 .unwrap();
1504
1505 assert!(
1506 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1507 "expected EarlyReturn{{status: new}} for simple complexity — free model should be tried first"
1508 );
1509 }
1510
1511 #[serial(cooldown_state)]
1512 #[tokio::test]
1513 async fn silent_exit0_retries_free_model_when_complexity_unknown() {
1514 cooldown::reset_global_state().await;
1515 let runner = MockRunner {
1516 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1517 };
1518 let err = AgentError::Unknown {
1519 exit_code: 0,
1520 message: String::new(),
1521 };
1522
1523 let result = handle_error(
1525 "test-1112-a2",
1526 &err,
1527 "opencode",
1528 &runner,
1529 Some("opencode/gpt-5.4-mini"),
1530 None,
1531 1,
1532 &None,
1533 "owner/repo",
1534 )
1535 .await
1536 .unwrap();
1537
1538 assert!(
1539 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1540 "expected EarlyReturn{{status: new}} for unknown complexity — free model should be tried first"
1541 );
1542 }
1543
1544 #[serial(cooldown_state)]
1545 #[tokio::test]
1546 async fn silent_exit0_retries_free_model_for_medium_complexity() {
1547 cooldown::reset_global_state().await;
1548 let runner = MockRunner {
1549 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1550 };
1551 let err = AgentError::Unknown {
1552 exit_code: 0,
1553 message: String::new(),
1554 };
1555
1556 let result = handle_error(
1558 "test-1195-medium",
1559 &err,
1560 "opencode",
1561 &runner,
1562 Some("opencode/copilot-gpt-5.4-mini"),
1563 Some("medium"),
1564 1,
1565 &None,
1566 "owner/repo",
1567 )
1568 .await
1569 .unwrap();
1570
1571 assert!(
1572 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1573 "expected EarlyReturn{{status: new}} for medium complexity — free model should be tried before failover"
1574 );
1575 }
1576
1577 #[serial(cooldown_state)]
1578 #[tokio::test]
1579 async fn silent_exit0_retries_free_model_for_complex_complexity() {
1580 cooldown::reset_global_state().await;
1581 let runner = MockRunner {
1582 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1583 };
1584 let err = AgentError::Unknown {
1585 exit_code: 0,
1586 message: String::new(),
1587 };
1588
1589 let result = handle_error(
1591 "test-1195-complex",
1592 &err,
1593 "opencode",
1594 &runner,
1595 Some("opencode/copilot-gpt-5.4-mini"),
1596 Some("complex"),
1597 1,
1598 &None,
1599 "owner/repo",
1600 )
1601 .await
1602 .unwrap();
1603
1604 assert!(
1605 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1606 "expected EarlyReturn{{status: new}} for complex complexity — free model should be tried before failover"
1607 );
1608 }
1609
1610 #[serial(cooldown_state)]
1613 #[tokio::test]
1614 async fn context_overflow_escalates_to_needs_review_immediately() {
1615 cooldown::reset_global_state().await;
1616 let runner = MockRunner { free: vec![] };
1617 let err = AgentError::ContextOverflow {
1618 message: "prompt is too long: 250000 tokens, max 200000".to_string(),
1619 };
1620
1621 let result = handle_error(
1622 "test-2129-a",
1623 &err,
1624 "claude",
1625 &runner,
1626 Some("sonnet"),
1627 Some("medium"),
1628 1,
1629 &None,
1630 "owner/repo",
1631 )
1632 .await
1633 .unwrap();
1634
1635 assert!(
1636 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "needs_review"),
1637 "context overflow must escalate directly to needs_review without cycling through fallback agents"
1638 );
1639 }
1640
1641 #[serial(cooldown_state)]
1644 #[tokio::test]
1645 async fn free_model_reroute_succeeds_without_store() {
1646 cooldown::reset_global_state().await;
1647 let runner = MockRunner {
1648 free: vec!["opencode/mimo-v2-omni-free".to_string()],
1649 };
1650 let err = AgentError::Unknown {
1651 exit_code: 0,
1652 message: String::new(),
1653 };
1654
1655 let result = handle_error(
1657 "test-2293-a",
1658 &err,
1659 "opencode",
1660 &runner,
1661 Some("opencode/gpt-5.4-mini"),
1662 Some("simple"),
1663 1,
1664 &None,
1665 "owner/repo",
1666 )
1667 .await
1668 .unwrap();
1669
1670 assert!(
1671 matches!(result, ErrorHandleResult::EarlyReturn { ref status } if status == "new"),
1672 "expected EarlyReturn{{status: new}} — store_set_result with None store should not prevent reroute"
1673 );
1674 }
1675
1676 #[serial(cooldown_state)]
1677 #[tokio::test]
1678 async fn silent_exit0_falls_through_when_no_free_models() {
1679 cooldown::reset_global_state().await;
1680 let runner = MockRunner { free: vec![] };
1681 let err = AgentError::Unknown {
1682 exit_code: 0,
1683 message: String::new(),
1684 };
1685
1686 let result = handle_error(
1687 "test-1112-b",
1688 &err,
1689 "opencode",
1690 &runner,
1691 Some("opencode/gpt-5.4-mini"),
1692 Some("simple"),
1693 1,
1694 &None,
1695 "owner/repo",
1696 )
1697 .await
1698 .unwrap();
1699
1700 assert!(
1701 matches!(result, ErrorHandleResult::Continue { .. }),
1702 "expected Continue (fallthrough to failover) when no free models available"
1703 );
1704 }
1705
1706 #[serial(cooldown_state)]
1712 #[tokio::test]
1713 async fn invalid_response_sets_model_cooldown() {
1714 cooldown::reset_global_state().await;
1715 let runner = MockRunner { free: vec![] };
1716 let agent = "opencode";
1717 let model = "opencode/nemotron-3-super-free";
1718
1719 assert!(
1721 !cooldown::is_model_in_cooldown(agent, model),
1722 "model should not be in cooldown before handle_error"
1723 );
1724
1725 let err = AgentError::InvalidResponse {
1726 raw: "invalid json output".to_string(),
1727 };
1728
1729 let _result = handle_error(
1730 "test-2750-invalid-response",
1731 &err,
1732 agent,
1733 &runner,
1734 Some(model),
1735 Some("simple"),
1736 1,
1737 &None,
1738 "owner/repo",
1739 )
1740 .await
1741 .unwrap();
1742
1743 assert!(
1745 cooldown::is_model_in_cooldown(agent, model),
1746 "model should be in cooldown after InvalidResponse (fixes issue #2750)"
1747 );
1748 }
1749
1750 #[serial(cooldown_state)]
1757 #[tokio::test]
1758 async fn billing_cycle_exhausted_with_known_model_sets_only_model_cooldown() {
1759 cooldown::reset_global_state().await;
1760 let runner = MockRunner { free: vec![] };
1761 let agent = "codex-3382-billing-cycle";
1762 let model = "gpt-5.5";
1763
1764 let err = AgentError::RateLimit {
1765 message: "rate limit: You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro).".to_string(),
1766 };
1767
1768 let _result = handle_error(
1769 "test-3382-a",
1770 &err,
1771 agent,
1772 &runner,
1773 Some(model),
1774 Some("medium"),
1775 1,
1776 &None,
1777 "owner/repo",
1778 )
1779 .await
1780 .unwrap();
1781
1782 assert!(
1784 !cooldown::is_agent_in_cooldown(agent),
1785 "agent must NOT be in agent-level cooldown when BillingCycleExhausted has a known model (issue #3382)"
1786 );
1787 assert!(
1789 cooldown::is_model_in_cooldown(agent, model),
1790 "model must be in cooldown after BillingCycleExhausted (issue #3382)"
1791 );
1792 assert!(
1794 !cooldown::is_model_in_cooldown(agent, "gpt-5.4"),
1795 "other models on the same agent must not be cooled by a single model's billing exhaustion"
1796 );
1797 }
1798
1799 #[serial(cooldown_state)]
1800 #[tokio::test]
1801 async fn summarize_rate_limit_error_extracts_api_retry_info() {
1802 cooldown::reset_global_state().await;
1803 let raw_output = r#"some log lines
1804{"type":"system","subtype":"api_retry","attempt":6,"max_retries":10,"retry_delay_ms":17266.789,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1805{"type":"system","subtype":"api_retry","attempt":7,"max_retries":10,"retry_delay_ms":35162.66,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1806more logs"#;
1807
1808 let summarized = super::summarize_rate_limit_error(raw_output);
1809 assert_eq!(summarized, "status=429 after 7 attempts (last delay 35s)");
1810 }
1811
1812 #[serial(cooldown_state)]
1813 #[tokio::test]
1814 async fn summarize_rate_limit_error_returns_original_when_no_api_retry() {
1815 cooldown::reset_global_state().await;
1816 let raw_output = "some regular error message without api_retry JSON";
1817
1818 let summarized = super::summarize_rate_limit_error(raw_output);
1819 assert_eq!(summarized, raw_output);
1820 }
1821
1822 #[serial(cooldown_state)]
1823 #[tokio::test]
1824 async fn summarize_rate_limit_error_handles_malformed_json_gracefully() {
1825 cooldown::reset_global_state().await;
1826 let raw_output = r#"some logs
1827{"type":"system","subtype":"api_retry","attempt":"not_a_number","retry_delay_ms":invalid,"error_status":429}
1828more logs"#;
1829
1830 let summarized = super::summarize_rate_limit_error(raw_output);
1831 assert!(summarized.starts_with("status="));
1833 assert!(summarized.contains("after"));
1834 assert!(summarized.contains("attempts"));
1835 assert!(summarized.contains("(last delay"));
1836 }
1837
1838 #[serial(cooldown_state)]
1839 #[tokio::test]
1840 async fn summarize_rate_limit_error_works_with_single_line() {
1841 cooldown::reset_global_state().await;
1842 let raw_output = r#"{"type":"system","subtype":"api_retry","attempt":3,"max_retries":10,"retry_delay_ms":5000,"error_status":429,"error":"rate_limit"}"#;
1843
1844 let summarized = super::summarize_rate_limit_error(raw_output);
1845 assert_eq!(summarized, "status=429 after 3 attempts (last delay 5s)");
1846 }
1847
1848 #[serial(cooldown_state)]
1852 #[tokio::test]
1853 async fn extract_json_field_handles_quoted_strings() {
1854 cooldown::reset_global_state().await;
1855 let json = r#"{"error_status":429,"attempt":7,"retry_delay_ms":35162.66,"error":"rate_limit","session_id":"331c099b-..."}"#;
1856
1857 assert_eq!(
1858 super::extract_json_field(json, "\"error\":"),
1859 Some("rate_limit".to_string()),
1860 "quoted string field mid-object should be extracted correctly"
1861 );
1862 assert_eq!(
1863 super::extract_json_field(json, "\"session_id\":"),
1864 Some("331c099b-...".to_string()),
1865 "quoted string field at end of object (no trailing comma) should be extracted"
1866 );
1867 assert_eq!(
1869 super::extract_json_field(json, "\"error_status\":"),
1870 Some("429".to_string())
1871 );
1872 assert_eq!(
1873 super::extract_json_field(json, "\"attempt\":"),
1874 Some("7".to_string())
1875 );
1876 assert_eq!(
1877 super::extract_json_field(json, "\"retry_delay_ms\":"),
1878 Some("35162.66".to_string())
1879 );
1880 }
1881
1882 #[serial(cooldown_state)]
1885 #[tokio::test]
1886 async fn summarize_rate_limit_error_handles_truncated_fragment_with_subtype() {
1887 cooldown::reset_global_state().await;
1888 let raw_output = r#"some log context
1890,"subtype":"api_retry","attempt":5,"retry_delay_ms":20000,"error_status":429,"error":"rate_limit"
1891more context"#;
1892
1893 let summarized = super::summarize_rate_limit_error(raw_output);
1894 assert!(
1895 !summarized.contains("subtype"),
1896 "summarized output must not contain raw 'subtype' JSON fragment"
1897 );
1898 assert!(
1899 !summarized.contains('{'),
1900 "summarized output must not contain JSON braces"
1901 );
1902 assert!(
1903 summarized.starts_with("status="),
1904 "summarized output should start with 'status=' — got: {summarized}"
1905 );
1906 }
1910
1911 #[serial(cooldown_state)]
1913 #[tokio::test]
1914 async fn summarize_rate_limit_error_handles_fragment_with_extractable_fields() {
1915 cooldown::reset_global_state().await;
1916 let raw_output = r#"log output
1917,"subtype":"api_retry","attempt":5,"retry_delay_ms":20000,"error_status":429,"error":"rate_limit","session_id":"abc123","max_retries":10}
1918rest of output"#;
1919
1920 let summarized = super::summarize_rate_limit_error(raw_output);
1921 assert_eq!(summarized, "status=429 after 5 attempts (last delay 20s)");
1922 }
1923
1924 #[serial(cooldown_state)]
1926 #[tokio::test]
1927 async fn summarize_rate_limit_error_complete_json_still_works() {
1928 cooldown::reset_global_state().await;
1929 let raw_output = r#"some log lines
1930{"type":"system","subtype":"api_retry","attempt":6,"max_retries":10,"retry_delay_ms":17266.789,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1931{"type":"system","subtype":"api_retry","attempt":7,"max_retries":10,"retry_delay_ms":35162.66,"error_status":429,"error":"rate_limit","session_id":"331c099b-..."}
1932more logs"#;
1933
1934 let summarized = super::summarize_rate_limit_error(raw_output);
1935 assert_eq!(summarized, "status=429 after 7 attempts (last delay 35s)");
1936 }
1937
1938 #[serial(cooldown_state)]
1940 #[tokio::test]
1941 async fn summarize_rate_limit_error_no_api_retry_no_raw_passthrough() {
1942 cooldown::reset_global_state().await;
1943 let raw_output = "some regular error message without api_retry JSON that is quite long and should be truncated";
1944
1945 let summarized = super::summarize_rate_limit_error(raw_output);
1946 assert!(
1948 summarized.len() <= 140,
1949 "expected truncated first-line, got: {summarized}"
1950 );
1951 }
1952
1953 #[serial(cooldown_state)]
1955 #[tokio::test]
1956 async fn summarize_rate_limit_error_no_api_retry_short() {
1957 cooldown::reset_global_state().await;
1958 let raw_output = "rate limit exceeded";
1959
1960 let summarized = super::summarize_rate_limit_error(raw_output);
1961 assert_eq!(summarized, raw_output);
1962 }
1963
1964 #[serial(cooldown_state)]
1970 #[tokio::test]
1971 async fn model_not_found_applies_persistent_cooldown() {
1972 cooldown::reset_global_state().await;
1973 let runner = MockRunner { free: vec![] };
1974 let agent = "opencode-2941-not-found";
1975 let model = "github-copilot/claude-opus-4.6";
1976 let key = format!("{agent}:{model}");
1977
1978 let err = AgentError::ModelUnavailable {
1979 model: model.to_string(),
1980 message: "Model not found: github-copilot/claude-opus-4.6".to_string(),
1981 };
1982
1983 let _result = handle_error(
1984 "test-2941-a",
1985 &err,
1986 agent,
1987 &runner,
1988 Some(model),
1989 Some("complex"),
1990 1,
1991 &None,
1992 "owner/repo",
1993 )
1994 .await
1995 .unwrap();
1996
1997 let now = chrono::Utc::now().timestamp();
1998 let until =
1999 cooldown::cooldown_until(&key).expect("model not found should set model cooldown");
2000 let remaining = until.saturating_sub(now);
2001
2002 assert!(
2005 remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2006 "expected ~4h (persistent) cooldown for 'Model not found', got {remaining}s — \
2007 may have applied transient 5min cooldown instead"
2008 );
2009 }
2010
2011 #[serial(cooldown_state)]
2018 #[tokio::test]
2019 async fn model_not_supported_applies_persistent_cooldown() {
2020 cooldown::reset_global_state().await;
2021 let runner = MockRunner { free: vec![] };
2022 let agent = "codex-3215-not-supported";
2023 let model = "gpt-5.2-codex";
2024 let key = format!("{agent}:{model}");
2025
2026 let err = AgentError::ModelUnavailable {
2027 model: model.to_string(),
2028 message: "The 'gpt-5.2-codex' model is not supported when using Codex with a ChatGPT account.".to_string(),
2029 };
2030
2031 let _result = handle_error(
2032 "test-3215-a",
2033 &err,
2034 agent,
2035 &runner,
2036 Some(model),
2037 Some("medium"),
2038 1,
2039 &None,
2040 "owner/repo",
2041 )
2042 .await
2043 .unwrap();
2044
2045 let now = chrono::Utc::now().timestamp();
2046 let until =
2047 cooldown::cooldown_until(&key).expect("model not supported should set model cooldown");
2048 let remaining = until.saturating_sub(now);
2049
2050 assert!(
2051 remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2052 "expected ~4h (persistent) cooldown for 'not supported', got {remaining}s — \
2053 may have applied transient 5min cooldown instead (issue #3215)"
2054 );
2055 }
2056
2057 #[serial(cooldown_state)]
2058 #[tokio::test]
2059 async fn model_not_available_in_country_applies_persistent_cooldown() {
2060 cooldown::reset_global_state().await;
2061 let runner = MockRunner { free: vec![] };
2062 let agent = "opencode-3650-country";
2063 let model = "some-region-locked-model";
2064 let key = format!("{agent}:{model}");
2065
2066 for message in [
2068 "This model is not available in your country.",
2069 "This model is not available in your region.",
2070 ] {
2071 cooldown::reset_global_state().await;
2072 let err = AgentError::ModelUnavailable {
2073 model: model.to_string(),
2074 message: message.to_string(),
2075 };
2076
2077 let _result = handle_error(
2078 "test-3650",
2079 &err,
2080 agent,
2081 &runner,
2082 Some(model),
2083 Some("medium"),
2084 1,
2085 &None,
2086 "owner/repo",
2087 )
2088 .await
2089 .unwrap();
2090
2091 let now = chrono::Utc::now().timestamp();
2092 let until =
2093 cooldown::cooldown_until(&key).expect("region block should set model cooldown");
2094 let remaining = until.saturating_sub(now);
2095
2096 assert!(
2097 remaining >= cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
2098 "expected ~4h (persistent) cooldown for '{message}', got {remaining}s — \
2099 may have applied transient 5min cooldown instead (issue #3650)"
2100 );
2101 }
2102 }
2103
2104 #[serial(cooldown_state)]
2110 #[tokio::test]
2111 async fn thinking_block_conflict_does_not_set_cooldown() {
2112 cooldown::reset_global_state().await;
2113 let runner = MockRunner { free: vec![] };
2114 let agent = "claude-3204-thinking-block";
2115 let model = "opus";
2116
2117 assert!(
2118 !cooldown::is_agent_in_cooldown(agent),
2119 "agent must not be in cooldown before handle_error"
2120 );
2121 assert!(
2122 !cooldown::is_model_in_cooldown(agent, model),
2123 "model must not be in cooldown before handle_error"
2124 );
2125
2126 let err = AgentError::ThinkingBlockConflict {
2127 message: "`thinking` or `redacted_thinking` blocks in the latest assistant message cannot be modified.".to_string(),
2128 };
2129
2130 let _result = handle_error(
2131 "test-3204-thinking",
2132 &err,
2133 agent,
2134 &runner,
2135 Some(model),
2136 Some("complex"),
2137 1,
2138 &None,
2139 "owner/repo",
2140 )
2141 .await
2142 .unwrap();
2143
2144 assert!(
2145 !cooldown::is_agent_in_cooldown(agent),
2146 "agent must NOT be in cooldown after ThinkingBlockConflict (fixes issue #3204)"
2147 );
2148 assert!(
2149 !cooldown::is_model_in_cooldown(agent, model),
2150 "model must NOT be in cooldown after ThinkingBlockConflict (fixes issue #3204)"
2151 );
2152 }
2153
2154 #[serial(cooldown_state)]
2156 #[tokio::test]
2157 async fn model_unavailable_transient_applies_short_cooldown() {
2158 cooldown::reset_global_state().await;
2159 let runner = MockRunner { free: vec![] };
2160 let agent = "opencode-2941-transient";
2161 let model = "github-copilot/gpt-4o";
2162 let key = format!("{agent}:{model}");
2163
2164 let err = AgentError::ModelUnavailable {
2165 model: model.to_string(),
2166 message: "No endpoints found for github-copilot/gpt-4o.".to_string(),
2167 };
2168
2169 let _result = handle_error(
2170 "test-2941-b",
2171 &err,
2172 agent,
2173 &runner,
2174 Some(model),
2175 Some("medium"),
2176 1,
2177 &None,
2178 "owner/repo",
2179 )
2180 .await
2181 .unwrap();
2182
2183 let now = chrono::Utc::now().timestamp();
2184 let until = cooldown::cooldown_until(&key)
2185 .expect("transient model unavailability should set model cooldown");
2186 let remaining = until.saturating_sub(now);
2187
2188 assert!(
2191 remaining < cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2192 "transient unavailability should use short cooldown (<4h), got {remaining}s"
2193 );
2194 }
2195
2196 #[serial(cooldown_state)]
2197 #[tokio::test]
2198 async fn model_unavailable_does_not_reuse_same_agent_when_agent_cooled() {
2199 cooldown::reset_global_state().await;
2200 let agent = "codex-3620";
2201 let failing_model = "gpt-5.4";
2202 let other_model = "gpt-5.5";
2203 let runner = MockRunnerWithModels {
2204 models: vec![failing_model.to_string(), other_model.to_string()],
2205 };
2206
2207 cooldown::set_agent_cooldown(agent, 3600).await;
2208
2209 let err = AgentError::ModelUnavailable {
2210 model: failing_model.to_string(),
2211 message: "model gpt-5.4 unavailable".to_string(),
2212 };
2213
2214 let result = handle_error(
2215 "test-3620",
2216 &err,
2217 agent,
2218 &runner,
2219 Some(failing_model),
2220 Some("medium"),
2221 1,
2222 &None,
2223 "owner/repo",
2224 )
2225 .await
2226 .unwrap();
2227
2228 if let ErrorHandleResult::EarlyReturn { status } = result {
2229 assert_ne!(
2230 status, "routed",
2231 "agent-wide cooldown was defeated by same-agent model substitution"
2232 );
2233 }
2234 }
2235}