1use crate::store;
10use crate::store::TaskStore;
11use std::collections::HashSet;
12use std::collections::hash_map::DefaultHasher;
13use std::fs;
14use std::fs::File;
15use std::hash::Hash;
16use std::hash::Hasher;
17use std::io::Read;
18use std::io::Seek;
19use std::io::SeekFrom;
20use std::path::{Path, PathBuf};
21use std::result;
22use std::sync::Arc;
23use std::thread;
24use std::time::Duration;
25use std::time::SystemTime;
26
27use tokio::fs as tokio_fs;
28use tokio::task;
29use tokio::time;
30use crate::cli::ndjson;
32use crate::cmd_cache;
33use crate::config as app_config;
34use crate::engine;
35pub use crate::engine::cooldown::{
36 clear_expired_cooldowns, is_agent_in_cooldown, is_model_in_cooldown,
37 record_agent_failure_with_message, record_model_failure, set_agent_cooldown,
38};
39use crate::engine::router::config;
40use crate::home;
41use crate::parser::AgentResponse;
42use crate::store::MemoryEntry;
43use crate::store::Task;
44
45const JITTER_FACTOR: f64 = 0.3;
46
47const MAX_OUTPUT_SIZE_BYTES: u64 = 10 * 1024 * 1024;
50
51fn read_file_with_limit<P: AsRef<Path>>(path: P, max_bytes: u64) -> anyhow::Result<String> {
57 let path = path.as_ref();
58 let metadata = fs::metadata(path)?;
59
60 if !metadata.is_file() {
61 anyhow::bail!("not a file: {}", path.display());
62 }
63
64 let file_size = metadata.len();
65
66 if file_size <= max_bytes {
68 return fs::read_to_string(path).map_err(Into::into);
69 }
70
71 let mut file = File::open(path)?;
73 let start_pos = file_size.saturating_sub(max_bytes);
74 file.seek(SeekFrom::Start(start_pos))?;
75
76 let mut buffer = Vec::with_capacity(usize::try_from(max_bytes).unwrap_or(0));
77 file.read_to_end(&mut buffer)?;
78
79 let content = String::from_utf8_lossy(&buffer);
81
82 tracing::warn!(
84 path = %path.display(),
85 file_size = file_size,
86 max_bytes = max_bytes,
87 "output file truncated: exceeded size limit"
88 );
89
90 Ok(content.into_owned())
91}
92
93pub(crate) fn calculate_backoff_delay(retry_count: usize) -> u64 {
98 let base_delay = config::retry_base_delay_ms();
99 let max_delay = config::retry_max_delay_ms();
100
101 let exponential =
103 base_delay * (2_u64.saturating_pow(u32::try_from(retry_count).unwrap_or(u32::MAX)));
104 let capped = exponential.min(max_delay);
105
106 #[expect(
111 clippy::cast_precision_loss,
112 clippy::cast_possible_truncation,
113 clippy::cast_sign_loss,
114 reason = "a jitter range in milliseconds, far below 2^52 and never negative"
115 )]
116 let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
117 let jitter = {
118 let mut h = DefaultHasher::new();
119 SystemTime::now().hash(&mut h);
120 thread::current().id().hash(&mut h);
121 h.finish() % (jitter_range * 2 + 1)
122 };
123
124 capped.saturating_sub(jitter_range) + jitter
125}
126
127fn fallback_retry_count(task: Option<&Task>) -> usize {
128 task.map_or(0, |t| {
129 let chain_retry_count = t
130 .limit_reroute_chain
131 .split(',')
132 .filter(|entry| !entry.is_empty())
133 .count();
134 let network_retry_count = usize::try_from(t.network_retries).unwrap_or(0);
135 chain_retry_count.max(network_retry_count)
136 })
137}
138
139pub async fn wait_for_fallback_backoff(
142 task_id: &str,
143 store: &Option<Arc<TaskStore>>,
144 repo: &str,
145) -> u64 {
146 let task = store::opt_store_get_task(store, repo, task_id).await;
147
148 let retry_count = fallback_retry_count(task.as_ref());
151
152 let delay = calculate_backoff_delay(retry_count);
153
154 if delay > 0 {
155 tracing::debug!(
156 task_id,
157 retry_count = retry_count,
158 delay_ms = delay,
159 "fallback backoff: waiting before retry"
160 );
161 time::sleep(Duration::from_millis(delay)).await;
162 }
163
164 delay
165}
166
167#[derive(Debug, Clone)]
172pub enum WeightSignal {
173 Success { agent: String },
175 RateLimited { agent: String },
177 Failed { agent: String },
180 Blocked,
182 Rerouted,
185 None,
187}
188
189pub async fn read_output_file(task_id: &str, primary_path: &Path, repo: &str) -> String {
194 if let Some(content) = task::spawn_blocking({
196 let p = primary_path.to_path_buf();
197 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
198 })
199 .await
200 .ok()
201 .and_then(result::Result::ok)
202 && !content.is_empty()
203 {
204 return content;
205 }
206
207 if let Ok(task_dir) = home::task_dir_async(repo, task_id).await {
209 let attempts_dir = task_dir.join("attempts");
210 if tokio_fs::metadata(&attempts_dir)
212 .await
213 .is_ok_and(|m| m.is_dir())
214 {
215 let mut attempt_nums: Vec<u32> = task::spawn_blocking({
216 let a = attempts_dir.clone();
217 move || {
218 match fs::read_dir(&a) {
219 Ok(entries) => {
220 let mut nums = Vec::new();
221 for entry in entries {
222 match entry {
223 Ok(e) => {
224 if let Some(n) =
225 e.file_name().to_str().and_then(|n| n.parse().ok())
226 {
227 nums.push(n);
228 }
229 }
230 Err(e) => tracing::warn!(
231 path = %a.display(),
232 err = %e,
233 "error reading attempt dir entry in output fallback"
234 ),
235 }
236 }
237 nums
238 }
239 Err(e) => {
240 tracing::warn!(path = %a.display(), err = %e, "failed to read attempts dir in output fallback");
241 Vec::new()
242 }
243 }
244 }
245 })
246 .await
247 .unwrap_or_default();
248
249 attempt_nums.sort_unstable_by(|a, b| b.cmp(a)); for n in attempt_nums {
251 let p = attempts_dir.join(n.to_string()).join("output.json");
252 if let Some(content) = task::spawn_blocking({
253 let p = p.clone();
254 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
255 })
256 .await
257 .ok()
258 .and_then(result::Result::ok)
259 && !content.is_empty()
260 {
261 tracing::info!(task_id, path = %p.display(), "read output from attempt dir");
262 return content;
263 }
264 }
265 }
266 }
267
268 let state_dir = home::state_dir().unwrap_or_else(|_| PathBuf::from("/tmp"));
270
271 let mut fallbacks = vec![
272 PathBuf::from(format!("/tmp/output-{task_id}.json")),
273 state_dir.join(format!("output-{task_id}.json")),
274 ];
275
276 if let Ok(legacy_path) = home::state_file(&format!("output-{task_id}.json"))
277 && !fallbacks.contains(&legacy_path)
278 {
279 fallbacks.push(legacy_path);
280 }
281
282 for path in &fallbacks {
283 if let Some(content) = task::spawn_blocking({
284 let p = path.clone();
285 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
286 })
287 .await
288 .ok()
289 .and_then(result::Result::ok)
290 && !content.is_empty()
291 {
292 tracing::info!(task_id, path = %path.display(), "read output from legacy fallback");
293 return content;
294 }
295 }
296
297 String::new()
298}
299
300pub fn pick_fallback_agent(
302 current_agent: &str,
303 chain: &str,
304 available_agents: &[String],
305) -> Option<String> {
306 clear_expired_cooldowns();
308
309 let chain_set: HashSet<&str> = if chain.is_empty() {
310 HashSet::new()
311 } else {
312 chain.split(',').collect()
313 };
314
315 for agent in available_agents {
319 if agent == current_agent || chain_set.contains(agent.as_str()) {
320 continue;
321 }
322 if is_agent_in_cooldown(agent) {
323 continue;
324 }
325 return Some(agent.clone());
332 }
333
334 None
335}
336
337#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum RetryableError {
340 Timeout,
342 Interrupted,
344 UsageLimit,
346 AuthError,
348 Failed,
350 MissingTooling,
352 ModelUnavailable,
356}
357
358impl RetryableError {
359 pub fn type_str(self) -> &'static str {
362 match self {
363 RetryableError::Timeout => "timeout",
364 RetryableError::Interrupted => "interrupted",
365 RetryableError::UsageLimit => "rate_limit",
366 RetryableError::AuthError => "auth_error",
367 RetryableError::ModelUnavailable => "model_unavailable",
368 RetryableError::Failed | RetryableError::MissingTooling => "failed",
369 }
370 }
371}
372
373pub async fn handle_failover(
382 task_id: &str,
383 agent_name: &str,
384 error_type: RetryableError,
385 error_message: &str,
386 credit_recorded: bool,
387 store: &Option<Arc<TaskStore>>,
388 repo: &str,
389) -> String {
390 let chain = get_reroute_chain(task_id, store, repo).await;
392 let chain = update_reroute_chain(task_id, agent_name, &chain, store, repo).await;
393
394 if error_type == RetryableError::Timeout
395 && let Some(status) =
396 block_after_repeated_timeouts(task_id, error_message, store, repo).await
397 {
398 return status;
399 }
400
401 let available: Vec<String> = engine::configured_agents()
403 .into_iter()
404 .filter(|a| cmd_cache::command_exists(a))
405 .collect();
406
407 let chain_set: HashSet<&str> = if chain.is_empty() {
409 HashSet::new()
410 } else {
411 chain.split(',').collect()
412 };
413 let is_exhausted = available.iter().all(|a| chain_set.contains(a.as_str()));
414
415 if is_exhausted {
416 let is_retryable = matches!(
422 error_type,
423 RetryableError::UsageLimit
424 | RetryableError::AuthError
425 | RetryableError::ModelUnavailable
426 | RetryableError::Timeout
427 | RetryableError::Interrupted
428 );
429 let next_status = if is_retryable { "new" } else { "blocked" };
430
431 tracing::warn!(
432 task_id,
433 agent = agent_name,
434 error_type = ?error_type,
435 next_status,
436 "all agents exhausted"
437 );
438
439 let mut store_fields: Vec<(&str, serde_json::Value)> = Vec::new();
440
441 if is_retryable {
442 let msg = format!("{error_message} (all agents exhausted, will retry when available)");
445 store_fields.push(("last_error", serde_json::json!(msg)));
446 store_fields.push(("limit_reroute_chain", serde_json::json!("")));
447 } else {
448 let msg = format!("{error_message} (all agents exhausted)");
449 let block_reason = format!(
450 "all agents exhausted: {} — {}",
451 error_type.type_str(),
452 error_message
453 );
454 store_fields.push(("last_error", serde_json::json!(msg)));
455 store_fields.push(("block_reason", serde_json::json!(block_reason)));
456 }
457
458 if let Err(e) = store::store_set_result(store, repo, task_id, &store_fields).await {
459 tracing::warn!(task_id, err = %e, "failed to write agents_exhausted fields to store");
460 }
461
462 let details = serde_json::json!({
464 "failure_reason": error_type.type_str(),
465 "error_message": error_message,
466 "chain": chain,
467 "agents_exhausted": true,
468 "is_retryable": is_retryable,
469 });
470 store::store_log_activity(
471 store,
472 repo,
473 task_id,
474 "rerouted",
475 Some("in_progress"),
476 Some(next_status),
477 None::<&str>,
478 None::<&str>,
479 Some(&details),
480 )
481 .await;
482
483 return next_status.to_string();
484 }
485
486 if let Some(next) = pick_fallback_agent(agent_name, &chain, &available) {
488 tracing::info!(
489 task_id,
490 from = agent_name,
491 to = next,
492 error_type = ?error_type,
493 "failover: switching to fallback agent"
494 );
495
496 if !matches!(
498 error_type,
499 RetryableError::MissingTooling
500 | RetryableError::ModelUnavailable
501 | RetryableError::Timeout
502 | RetryableError::Interrupted
503 ) {
504 if !credit_recorded {
507 record_agent_failure_with_message(agent_name, error_message).await;
508 }
509
510 set_agent_cooldown(agent_name, 120).await;
515 }
516
517 let msg = format!("{error_message}, clearing agent/model for re-route");
518 if let Err(e) = store::store_set_result(
519 store,
520 repo,
521 task_id,
522 &[
523 ("agent", serde_json::json!("")),
524 ("model", serde_json::json!("")),
525 ("last_error", serde_json::json!(msg)),
526 ],
527 )
528 .await
529 {
530 tracing::warn!(task_id, err = %e, "failed to write agent/model clear to store");
531 }
532
533 wait_for_fallback_backoff(task_id, store, repo).await;
539
540 let details = serde_json::json!({
542 "failure_reason": error_type.type_str(),
543 "error_message": error_message,
544 "from_agent": agent_name,
545 "to_agent": next,
546 "chain": chain,
547 });
548 store::store_log_activity(
549 store,
550 repo,
551 task_id,
552 "rerouted",
553 Some("in_progress"),
554 Some("new"),
555 Some(&next),
556 None::<&str>,
557 Some(&details),
558 )
559 .await;
560
561 return "new".to_string();
562 }
563
564 tracing::warn!(task_id, agent = agent_name, "no fallback agents available");
566
567 let is_retryable = matches!(
568 error_type,
569 RetryableError::UsageLimit
570 | RetryableError::AuthError
571 | RetryableError::ModelUnavailable
572 | RetryableError::Timeout
573 | RetryableError::Interrupted
574 );
575
576 if !matches!(
578 error_type,
579 RetryableError::MissingTooling
580 | RetryableError::ModelUnavailable
581 | RetryableError::Timeout
582 | RetryableError::Interrupted
583 ) && !credit_recorded
584 {
585 record_agent_failure_with_message(agent_name, error_message).await;
586 }
587
588 if is_retryable {
589 wait_for_fallback_backoff(task_id, store, repo).await;
592
593 let msg = format!("{error_message} (no fallback agents, will retry)");
594 if let Err(e) = store::store_set_result(
595 store,
596 repo,
597 task_id,
598 &[
599 ("last_error", serde_json::json!(msg)),
600 ("limit_reroute_chain", serde_json::json!("")),
601 ],
602 )
603 .await
604 {
605 tracing::warn!(task_id, err = %e, "failed to write no_fallback retry fields to store");
606 }
607
608 let details = serde_json::json!({
609 "failure_reason": error_type.type_str(),
610 "error_message": error_message,
611 "chain": chain,
612 "no_fallback_available": true,
613 "is_retryable": true,
614 });
615 store::store_log_activity(
616 store,
617 repo,
618 task_id,
619 "rerouted",
620 Some("in_progress"),
621 Some("new"),
622 None::<&str>,
623 None::<&str>,
624 Some(&details),
625 )
626 .await;
627
628 return "new".to_string();
629 }
630
631 let msg = format!("{error_message}, no fallback agents");
632 if let Err(e) = store::store_set_result(
633 store,
634 repo,
635 task_id,
636 &[("last_error", serde_json::json!(msg))],
637 )
638 .await
639 {
640 tracing::warn!(task_id, err = %e, "failed to write no_fallback last_error to store");
641 }
642
643 let details = serde_json::json!({
645 "failure_reason": error_type.type_str(),
646 "error_message": error_message,
647 "chain": chain,
648 "no_fallback_available": true,
649 });
650 store::store_log_activity(
651 store,
652 repo,
653 task_id,
654 "rerouted",
655 Some("in_progress"),
656 Some("needs_review"),
657 None::<&str>,
658 None::<&str>,
659 Some(&details),
660 )
661 .await;
662
663 "needs_review".to_string()
664}
665
666pub async fn block_after_repeated_timeouts(
669 task_id: &str,
670 error_message: &str,
671 store: &Option<Arc<TaskStore>>,
672 repo: &str,
673) -> Option<String> {
674 let max: i64 = app_config::get("workflow.max_timeout_retries")
675 .ok()
676 .and_then(|v| v.parse().ok())
677 .unwrap_or(3);
678 let count = match store::store_increment(store, repo, task_id, "timeout_retries").await {
679 Ok(count) => i64::try_from(count).unwrap_or(i64::MAX),
680 Err(e) => {
681 tracing::warn!(task_id, err = %e, "failed to increment timeout_retries");
682 return None;
683 }
684 };
685 if count < max {
686 return None;
687 }
688
689 let block_reason = format!("timed out {count} times");
690 tracing::warn!(
691 task_id,
692 count,
693 max,
694 "timeout retries exhausted, blocking task"
695 );
696 if let Err(e) = store::store_set_result(
697 store,
698 repo,
699 task_id,
700 &[
701 (
702 "last_error",
703 serde_json::json!(format!("{error_message} ({block_reason})")),
704 ),
705 ("block_reason", serde_json::json!(block_reason)),
706 ("limit_reroute_chain", serde_json::json!("")),
707 ],
708 )
709 .await
710 {
711 tracing::warn!(task_id, err = %e, "failed to write timeout block fields to store");
712 }
713 let details = serde_json::json!({
714 "failure_reason": "timeout",
715 "error_message": error_message,
716 "timeout_retries": count,
717 });
718 store::store_log_activity(
719 store,
720 repo,
721 task_id,
722 "rerouted",
723 Some("in_progress"),
724 Some("blocked"),
725 None::<&str>,
726 None::<&str>,
727 Some(&details),
728 )
729 .await;
730 Some("blocked".to_string())
731}
732
733pub async fn get_reroute_chain(
735 task_id: &str,
736 store: &Option<Arc<TaskStore>>,
737 repo: &str,
738) -> String {
739 store::opt_store_get_task(store, repo, task_id)
740 .await
741 .map(|t| t.limit_reroute_chain)
742 .unwrap_or_default()
743 .trim()
744 .to_string()
745}
746
747pub async fn update_reroute_chain(
749 task_id: &str,
750 current_agent: &str,
751 existing_chain: &str,
752 store: &Option<Arc<TaskStore>>,
753 repo: &str,
754) -> String {
755 let mut chain = existing_chain.to_string();
756 if chain.is_empty() {
757 chain = current_agent.to_string();
758 } else if !chain.split(',').any(|a| a == current_agent) {
759 chain = format!("{chain},{current_agent}");
760 }
761
762 if let Err(e) = store::store_set_result(
763 store,
764 repo,
765 task_id,
766 &[("limit_reroute_chain", serde_json::json!(chain))],
767 )
768 .await
769 {
770 tracing::warn!(task_id, err = %e, "failed to write limit_reroute_chain to store");
771 }
772 chain
773}
774
775#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
777pub struct ReviewResponse {
778 pub decision: String,
780 pub notes: String,
782 pub test_results: Option<String>,
784 pub issues: Vec<ReviewIssue>,
786}
787
788#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
790pub struct ReviewIssue {
791 pub file: String,
793 pub line: Option<u32>,
795 pub severity: String,
797 pub description: String,
799}
800
801pub fn parse_review_response(text: &str) -> anyhow::Result<ReviewResponse> {
807 let resp = parse_review_json(text)?;
808 if matches!(resp.decision.as_str(), "approve" | "request_changes") {
809 return Ok(resp);
810 }
811
812 let stripped = text.replace("approve|request_changes", "");
815 infer_review_response_from_text(&stripped)
816 .ok_or_else(|| anyhow::anyhow!("invalid review decision: {}", resp.decision))
817}
818
819fn parse_review_json(text: &str) -> anyhow::Result<ReviewResponse> {
820 if let Ok(resp) = serde_json::from_str::<ReviewResponse>(text) {
822 return Ok(resp);
823 }
824
825 if let Some(json_str) = extract_json_block(text)
827 && let Ok(resp) = serde_json::from_str::<ReviewResponse>(&json_str)
828 {
829 return Ok(resp);
830 }
831
832 if let Some(resp) = extract_review_response_object(text) {
833 return Ok(resp);
834 }
835
836 anyhow::bail!("failed to parse review response")
837}
838
839fn infer_review_response_from_text(text: &str) -> Option<ReviewResponse> {
840 let lower = text.to_ascii_lowercase();
841 let has_changes_requested =
842 lower.contains("changes requested") || lower.contains("request_changes");
843 if has_changes_requested {
844 return Some(ReviewResponse {
845 decision: "request_changes".to_string(),
846 notes: "Inferred decision from plain-text review output.".to_string(),
847 test_results: None,
848 issues: Vec::new(),
849 });
850 }
851
852 let positive_approval = lower
855 .split(|c: char| !c.is_ascii_alphabetic())
856 .any(|word| word == "lgtm")
857 || lower.contains("decision is `approve`")
858 || lower.contains("decision: `approve`")
859 || lower.contains("decision is approve")
860 || lower.contains("decision: approve");
861 if positive_approval {
862 return Some(ReviewResponse {
863 decision: "approve".to_string(),
864 notes: "Inferred decision from plain-text review output.".to_string(),
865 test_results: None,
866 issues: Vec::new(),
867 });
868 }
869
870 None
871}
872
873pub fn infer_review_response(text: &str) -> Option<ReviewResponse> {
877 infer_review_response_from_text(text)
878}
879
880fn extract_json_block(text: &str) -> Option<String> {
885 let mut search_from = 0;
886 while let Some(start) = text[search_from..].find("```json") {
887 let abs_start = search_from + start;
888 let after_tag = abs_start + "```json".len();
889 let Some(rest) = text[after_tag..].strip_prefix('\n') else {
890 search_from = after_tag;
891 continue;
892 };
893 let content_start = text.len() - rest.len();
894 let Some(end_offset) = rest.find("```") else {
895 search_from = content_start;
896 continue;
897 };
898 let end = content_start + end_offset;
899 let content = text[content_start..end].trim();
900 if content.starts_with('{') {
901 return Some(content.to_string());
902 }
903 search_from = end + 3;
904 }
905 None
906}
907
908fn extract_review_response_object(text: &str) -> Option<ReviewResponse> {
912 let mut start: Option<usize> = None;
913 let mut depth: i32 = 0;
914 let mut in_string = false;
915 let mut escape = false;
916
917 for (idx, ch) in text.char_indices() {
918 if start.is_some() {
919 if in_string {
920 if escape {
921 escape = false;
922 } else if ch == '\\' {
923 escape = true;
924 } else if ch == '"' {
925 in_string = false;
926 }
927 continue;
928 }
929
930 match ch {
931 '"' => in_string = true,
932 '{' => depth += 1,
933 '}' => {
934 depth -= 1;
935 if depth == 0 {
936 debug_assert!(
937 start.is_some(),
938 "start must be Some when depth returns to 0"
939 );
940 let start_idx = start.unwrap_or(0);
941 let candidate = &text[start_idx..=idx];
942 if let Ok(resp) = serde_json::from_str::<ReviewResponse>(candidate) {
943 return Some(resp);
944 }
945 start = None;
946 }
947 }
948 _ => {}
949 }
950 continue;
951 }
952
953 if ch == '{' {
954 start = Some(idx);
955 depth = 1;
956 in_string = false;
957 escape = false;
958 }
959 }
960
961 None
962}
963
964#[expect(
966 clippy::too_many_arguments,
967 reason = "a memory entry records the task, attempt, agent, model and outcome"
968)]
969pub async fn store_learnings_from_response(
970 task_id: &str,
971 attempt: u32,
972 agent: &str,
973 model: Option<&str>,
974 response: &AgentResponse,
975 error: Option<&str>,
976 store: &Option<Arc<TaskStore>>,
977 repo: &str,
978 raw_stdout: &str,
979) {
980 let files_modified = if response.files.is_empty() {
982 let extracted = ndjson::extract_files_changed(raw_stdout);
983 if !extracted.is_empty() {
984 tracing::debug!(
985 task_id,
986 count = extracted.len(),
987 "files_modified: fell back to NDJSON extraction (agent reported none)"
988 );
989 }
990 extracted
991 } else {
992 response.files.clone()
993 };
994
995 let entry = MemoryEntry {
997 attempt,
998 agent: agent.to_string(),
999 model: model.map(String::from),
1000 learnings: response.learnings.clone(),
1001 error: error.map(String::from),
1002 files_modified,
1003 approach: response.summary.clone(),
1004 timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1005 };
1006
1007 if let Some(st) = store
1008 && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1009 {
1010 match st.append_memory(store_id, &entry).await {
1011 Err(e) => {
1012 tracing::warn!(task_id, error = ?e, "failed to store memory");
1013 }
1014 _ => {
1015 tracing::debug!(task_id, attempt, "stored memory for attempt");
1016 }
1017 }
1018 }
1019}
1020
1021pub async fn store_failure_memory(
1023 task_id: &str,
1024 attempt: u32,
1025 agent: &str,
1026 model: Option<&str>,
1027 error: &str,
1028 store: &Option<Arc<TaskStore>>,
1029 repo: &str,
1030) {
1031 let entry = MemoryEntry {
1032 attempt,
1033 agent: agent.to_string(),
1034 model: model.map(String::from),
1035 learnings: vec![],
1036 error: Some(error.to_string()),
1037 files_modified: vec![],
1038 approach: String::new(),
1039 timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1040 };
1041
1042 if let Some(st) = store
1043 && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1044 {
1045 match st.append_memory(store_id, &entry).await {
1046 Err(e) => {
1047 tracing::warn!(task_id, error = ?e, "failed to store failure memory");
1048 }
1049 _ => {
1050 tracing::debug!(task_id, attempt, "stored failure memory");
1051 }
1052 }
1053 }
1054}
1055
1056#[cfg(test)]
1057mod tests {
1058 use std::env;
1059 #[cfg(unix)]
1060 use std::os::unix::fs::PermissionsExt;
1061 use std::process;
1062
1063 use super::*;
1064 use crate::engine::cooldown;
1065 use crate::engine::runner::agents::{AgentError, find_agent_result, patterns};
1066 use crate::store::TaskStatus;
1067 use serial_test::serial;
1068
1069 #[serial(cooldown_state)]
1070 #[test]
1071 fn calculate_backoff_delay_jitter_is_centered() {
1072 let base = config::retry_base_delay_ms();
1076 let max = config::retry_max_delay_ms();
1077
1078 let capped = base.min(max);
1080 #[expect(
1081 clippy::cast_precision_loss,
1082 clippy::cast_possible_truncation,
1083 clippy::cast_sign_loss,
1084 reason = "mirrors the jitter range in calculate_backoff_delay"
1085 )]
1086 let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
1087 let lo = capped.saturating_sub(jitter_range);
1088 let hi = capped + jitter_range;
1089
1090 for _ in 0..200 {
1091 let delay = calculate_backoff_delay(0);
1092 assert!(
1093 delay >= lo && delay <= hi,
1094 "delay {delay} out of expected range [{lo}, {hi}]"
1095 );
1096 }
1097 }
1098
1099 #[serial(cooldown_state)]
1100 #[test]
1101 fn fallback_retry_count_uses_larger_network_retry_streak() {
1102 let task = Task {
1103 id: 1,
1104 external_id: None,
1105 repo: "owner/repo".to_string(),
1106 origin: "internal".to_string(),
1107 title: "test".to_string(),
1108 body: String::new(),
1109 status: TaskStatus::New,
1110 source: String::new(),
1111 source_id: String::new(),
1112 author: String::new(),
1113 url: String::new(),
1114 labels: vec![],
1115 agent: None,
1116 model: None,
1117 complexity: "simple".to_string(),
1118 estimate: 0,
1119 route_reason: String::new(),
1120 agent_profile: String::new(),
1121 selected_skills: String::new(),
1122 route_attempts: 0,
1123 attempts: 0,
1124 branch: String::new(),
1125 worktree: String::new(),
1126 worktree_cleaned: false,
1127 owner_node: None,
1128 summary: String::new(),
1129 last_error: String::new(),
1130 parent_id: None,
1131 block_reason: None,
1132 pr_number: None,
1133 pr_review_context: String::new(),
1134 last_review_ts: String::new(),
1135 review_ts_map: "{}".to_string(),
1136 last_comment_review_ts: String::new(),
1137 merge_conflict_retries: 0,
1138 ci_merge_failures: 0,
1139 pr_create_failures: 0,
1140 push_failures: 0,
1141 network_retries: 3,
1142 timeout_retries: 0,
1143 review_agent_failures: 0,
1144 review_resets: 0,
1145 review_cycles: 0,
1146 review_invocations: 0,
1147 needs_review_refires: 0,
1148 input_tokens: 0,
1149 output_tokens: 0,
1150 total_cost_usd: 0.0,
1151 model_reroute_chain: String::new(),
1152 limit_reroute_chain: "claude".to_string(),
1153 memory: vec![],
1154 delegations: vec![],
1155 auto_unblock_count: 0,
1156 auto_unblock_last_at: String::new(),
1157 auto_unblock_last_reason: String::new(),
1158 ci_recovery_count: 0,
1159 no_code_reroutes: 0,
1160 no_code_last_agent: String::new(),
1161 created_at: String::new(),
1162 updated_at: String::new(),
1163 };
1164
1165 assert_eq!(fallback_retry_count(Some(&task)), 3);
1166 }
1167
1168 #[serial(cooldown_state)]
1169 #[test]
1170 fn retryable_type_str_returns_correct_labels() {
1171 assert_eq!(RetryableError::Timeout.type_str(), "timeout");
1172 assert_eq!(RetryableError::Interrupted.type_str(), "interrupted");
1173 assert_eq!(RetryableError::UsageLimit.type_str(), "rate_limit");
1174 assert_eq!(RetryableError::AuthError.type_str(), "auth_error");
1175 assert_eq!(RetryableError::Failed.type_str(), "failed");
1176 assert_eq!(RetryableError::MissingTooling.type_str(), "failed");
1177 }
1178
1179 #[serial(cooldown_state)]
1180 #[test]
1181 fn is_usage_limit_detects_rate_limit() {
1182 assert!(patterns::detect_rate_limit("Error: rate limit exceeded").is_some());
1183 assert!(patterns::detect_rate_limit("HTTP 429 Too Many Requests").is_some());
1184 assert!(patterns::detect_rate_limit("quota exceeded for model").is_some());
1185 assert!(patterns::detect_context_overflow("context_length_exceeded").is_some());
1187 }
1188
1189 #[serial(cooldown_state)]
1190 #[test]
1191 fn is_usage_limit_rejects_normal_text() {
1192 assert!(patterns::detect_rate_limit("task completed successfully").is_none());
1193 assert!(patterns::detect_rate_limit("").is_none());
1194 }
1195
1196 #[serial(cooldown_state)]
1197 #[test]
1198 fn is_auth_error_detects_common_patterns() {
1199 assert!(patterns::detect_auth_error("401 Unauthorized").is_some());
1200 assert!(patterns::detect_auth_error("invalid api key provided").is_some());
1201 assert!(patterns::detect_auth_error("Your billing plan has expired").is_some());
1202 assert!(patterns::detect_auth_error("Error: 403 Forbidden").is_some());
1203 }
1204
1205 #[serial(cooldown_state)]
1206 #[test]
1207 fn is_auth_error_rejects_normal_text() {
1208 assert!(patterns::detect_auth_error("task completed successfully").is_none());
1209 assert!(patterns::detect_auth_error("").is_none());
1210 }
1211
1212 #[serial(cooldown_state)]
1213 #[test]
1214 fn detect_missing_tooling_finds_known_tools() {
1215 let result = patterns::detect_missing_tool("bun: command not found");
1216 assert!(result.is_some());
1217 if let AgentError::MissingTool { tool } = result.unwrap() {
1218 assert_eq!(tool, "bun");
1219 } else {
1220 panic!("expected MissingTool error");
1221 }
1222
1223 let result = patterns::detect_missing_tool("env: anchor: no such file");
1224 assert!(result.is_some());
1225 if let AgentError::MissingTool { tool } = result.unwrap() {
1226 assert_eq!(tool, "anchor");
1227 } else {
1228 panic!("expected MissingTool error");
1229 }
1230
1231 let result = patterns::detect_missing_tool("spawn docker enoent");
1232 assert!(result.is_some());
1233 if let AgentError::MissingTool { tool } = result.unwrap() {
1234 assert_eq!(tool, "docker");
1235 } else {
1236 panic!("expected MissingTool error");
1237 }
1238 }
1239
1240 #[serial(cooldown_state)]
1241 #[test]
1242 fn detect_missing_tooling_returns_none_for_normal() {
1243 assert!(patterns::detect_missing_tool("everything works fine").is_none());
1244 assert!(patterns::detect_missing_tool("").is_none());
1245 }
1246
1247 #[serial(cooldown_state)]
1250 #[tokio::test]
1251 async fn record_and_check_agent_cooldown() {
1252 cooldown::reset_global_state().await;
1253 let agent = "test_cooldown_agent_1";
1255 assert!(!is_agent_in_cooldown(agent));
1256 record_agent_failure_with_message(agent, "").await;
1257 assert!(is_agent_in_cooldown(agent));
1258 }
1259
1260 #[serial(cooldown_state)]
1261 #[tokio::test]
1262 async fn record_and_check_model_cooldown() {
1263 cooldown::reset_global_state().await;
1264 let agent = "test_cooldown_agent_2";
1265 let model = "test_model_x";
1266 assert!(!is_model_in_cooldown(agent, model));
1267 record_model_failure(agent, model).await;
1268 assert!(is_model_in_cooldown(agent, model));
1269 assert!(!is_model_in_cooldown(agent, "other_model"));
1271 }
1272
1273 #[serial(cooldown_state)]
1274 #[tokio::test]
1275 async fn clear_expired_does_not_remove_fresh_entries() {
1276 cooldown::reset_global_state().await;
1277 let agent = "test_cooldown_agent_3";
1278 record_agent_failure_with_message(agent, "").await;
1279 clear_expired_cooldowns();
1280 assert!(is_agent_in_cooldown(agent));
1282 }
1283
1284 #[serial(cooldown_state)]
1286 #[test]
1287 fn pick_fallback_skips_current_agent() {
1288 let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1289 let result = pick_fallback_agent("test_agent_a", "", &available);
1290 assert_eq!(result, Some("test_agent_b".to_string()));
1291 }
1292
1293 #[serial(cooldown_state)]
1294 #[test]
1295 fn pick_fallback_skips_chain_agents() {
1296 let available = vec![
1297 "test_agent_a".to_string(),
1298 "test_agent_b".to_string(),
1299 "test_agent_c".to_string(),
1300 ];
1301 let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1302 assert_eq!(result, Some("test_agent_c".to_string()));
1303 }
1304
1305 #[serial(cooldown_state)]
1306 #[test]
1307 fn pick_fallback_returns_none_when_exhausted() {
1308 let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1309 let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1310 assert!(result.is_none());
1311 }
1312
1313 #[serial(cooldown_state)]
1314 #[test]
1315 fn weight_signal_variants() {
1316 let success = WeightSignal::Success {
1317 agent: "claude".to_string(),
1318 };
1319 let limited = WeightSignal::RateLimited {
1320 agent: "codex".to_string(),
1321 };
1322 let none = WeightSignal::None;
1323
1324 assert!(format!("{success:?}").contains("claude"));
1326 assert!(format!("{limited:?}").contains("codex"));
1327 assert!(format!("{none:?}").contains("None"));
1328 }
1329
1330 #[serial(cooldown_state)]
1331 #[test]
1332 fn parse_review_response_direct_json() {
1333 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1334 let resp = parse_review_response(json).unwrap();
1335 assert_eq!(resp.decision, "approve");
1336 assert_eq!(resp.notes, "LGTM");
1337 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1338 }
1339
1340 #[serial(cooldown_state)]
1341 #[test]
1342 fn parse_review_response_from_markdown() {
1343 let md = r#"Here is my review:
1344
1345```json
1346{"decision":"request_changes","notes":"Fix the bug","issues":[{"file":"src/main.rs","line":10,"severity":"error","description":"null deref"}]}
1347```
1348
1349That's all."#;
1350 let resp = parse_review_response(md).unwrap();
1351 assert_eq!(resp.decision, "request_changes");
1352 assert_eq!(resp.issues.len(), 1);
1353 assert_eq!(resp.issues[0].file, "src/main.rs");
1354 }
1355
1356 #[serial(cooldown_state)]
1357 #[test]
1358 fn parse_review_response_embedded_json_object() {
1359 let text = "Review complete.\n\n{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}\nThanks!";
1360 let resp = parse_review_response(text).unwrap();
1361 assert_eq!(resp.decision, "approve");
1362 assert_eq!(resp.notes, "LGTM");
1363 }
1364
1365 #[serial(cooldown_state)]
1366 #[test]
1367 fn parse_review_response_template_echo_infers_from_notes() {
1368 let json = r#"{"decision":"approve|request_changes","notes":"LGTM","issues":[]}"#;
1369 assert_eq!(parse_review_response(json).unwrap().decision, "approve");
1370
1371 let no_signal = r#"{"decision":"approve|request_changes","notes":"hmm","issues":[]}"#;
1372 assert!(parse_review_response(no_signal).is_err());
1373 }
1374
1375 #[serial(cooldown_state)]
1376 #[test]
1377 fn parse_review_response_invalid() {
1378 let result = parse_review_response("not json at all");
1379 assert!(result.is_err());
1380 }
1381
1382 #[serial(cooldown_state)]
1383 #[test]
1384 fn extract_json_block_basic() {
1385 let md = "text\n```json\n{\"key\": \"value\"}\n```\nmore";
1386 let result = extract_json_block(md);
1387 assert!(result.is_some());
1388 assert!(result.unwrap().contains("\"key\""));
1389 }
1390
1391 #[serial(cooldown_state)]
1392 #[test]
1393 fn extract_json_block_skips_non_json_blocks() {
1394 let md = "```json\nnot-json-array\n```\n\n```json\n{\"real\": true}\n```";
1395 let result = extract_json_block(md);
1396 assert!(result.is_some());
1397 assert!(result.unwrap().contains("\"real\""));
1398 }
1399
1400 #[serial(cooldown_state)]
1401 #[test]
1402 fn extract_json_block_none_when_missing() {
1403 assert!(extract_json_block("no code blocks here").is_none());
1404 }
1405
1406 #[serial(cooldown_state)]
1407 #[test]
1408 fn extract_json_block_skips_malformed_intermediate_blocks() {
1409 let md = "prefix ```json{\"broken\": true}\n\n```json\n{\"real\": true}\n```";
1410 let result = extract_json_block(md);
1411 assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1412 }
1413
1414 #[serial(cooldown_state)]
1415 #[test]
1416 fn extract_json_block_skips_multiple_malformed_intermediate_blocks() {
1417 let md = "prefix ```json{\"broken\": true}\nmid ```json[1, 2, 3]\n\n```json\n{\"real\": true}\n```";
1418 let result = extract_json_block(md);
1419 assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1420 }
1421
1422 fn parse_review_plain(text: &str) -> anyhow::Result<ReviewResponse> {
1427 parse_review_response(text).or_else(|_| {
1428 infer_review_response(text)
1429 .ok_or_else(|| anyhow::anyhow!("failed to parse review response"))
1430 })
1431 }
1432
1433 #[serial(cooldown_state)]
1434 #[test]
1435 fn parse_review_from_output_direct_json() {
1436 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1437 let resp = parse_review_response(json).unwrap();
1438 assert_eq!(resp.decision, "approve");
1439 }
1440
1441 #[serial(cooldown_state)]
1442 #[test]
1443 fn parse_review_from_output_markdown() {
1444 let md = "Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}\n```\n";
1445 let resp = parse_review_response(md).unwrap();
1446 assert_eq!(resp.decision, "request_changes");
1447 }
1448
1449 #[serial(cooldown_state)]
1450 #[test]
1451 fn parse_review_from_output_plain_text_fallback() {
1452 let text = "The review is done. LGTM.";
1453 let resp = parse_review_plain(text).unwrap();
1454 assert_eq!(resp.decision, "approve");
1455 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1456 }
1457
1458 #[serial(cooldown_state)]
1460 #[test]
1461 fn parse_review_from_output_ndjson_part_format() {
1462 let ndjson = concat!(
1463 r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1464 "\n",
1465 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Reviewing the PR...\n\n```json\n{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1466 "\n",
1467 r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop"}}"#,
1468 );
1469 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1470 assert_eq!(resp.decision, "approve");
1471 assert_eq!(resp.notes, "All checks pass");
1472 }
1473
1474 #[serial(cooldown_state)]
1476 #[test]
1477 fn parse_review_from_output_ndjson_direct_text_format() {
1478 let ndjson = concat!(
1479 r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1480 "\n",
1481 r#"{"type":"text","timestamp":1001,"sessionID":"ses_abc","text":"Here is my review:\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the bug\",\"test_results\":\"fail\",\"issues\":[]}\n```\n"}"#,
1482 "\n",
1483 r#"{"type":"step_finish","timestamp":1002,"sessionID":"ses_abc"}"#,
1484 );
1485 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1486 assert_eq!(resp.decision, "request_changes");
1487 assert_eq!(resp.notes, "Fix the bug");
1488 }
1489
1490 #[serial(cooldown_state)]
1492 #[test]
1493 fn parse_review_from_output_ndjson_direct_json_in_text_event() {
1494 let ndjson = concat!(
1495 r#"{"type":"step_start","timestamp":1000}"#,
1496 "\n",
1497 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1498 "\n",
1499 r#"{"type":"step_finish","timestamp":1002}"#,
1500 );
1501 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1502 assert_eq!(resp.decision, "approve");
1503 }
1504
1505 #[serial(cooldown_state)]
1506 #[test]
1507 fn parse_review_from_output_ndjson_prefers_final_json_text_event() {
1508 let ndjson = concat!(
1509 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Working on it..."}}"#,
1510 "\n",
1511 r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1512 );
1513 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1514 assert_eq!(resp.decision, "approve");
1515 assert_eq!(resp.notes, "LGTM");
1516 }
1517
1518 #[serial(cooldown_state)]
1520 #[test]
1521 fn parse_review_from_output_ndjson_concatenated_text() {
1522 let ndjson = concat!(
1523 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Here is my\n"}}"#,
1524 "\n",
1525 r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"review:\n\n```json\n{\"decision\":\"approve\",\"notes\":\"OK\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1526 );
1527 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1528 assert_eq!(resp.decision, "approve");
1529 }
1530
1531 #[serial(cooldown_state)]
1536 #[test]
1537 fn parse_review_from_output_codex_agent_message() {
1538 let ndjson = concat!(
1539 r#"{"type":"thread.started","thread_id":"t1"}"#,
1540 "\n",
1541 r#"{"type":"turn.started"}"#,
1542 "\n",
1543 r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Planning git commands..."}}"#,
1544 "\n",
1545 r#"{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"git log --oneline -5","output":"abc123 fix: something"}}"#,
1546 "\n",
1547 r#"{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1548 "\n",
1549 r#"{"type":"turn.completed"}"#,
1550 );
1551 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1552 assert_eq!(resp.decision, "approve");
1553 assert_eq!(resp.notes, "All checks pass");
1554 }
1555
1556 #[serial(cooldown_state)]
1558 #[test]
1559 fn parse_review_from_output_codex_agent_message_markdown() {
1560 let ndjson = concat!(
1561 r#"{"type":"thread.started","thread_id":"t1"}"#,
1562 "\n",
1563 r#"{"type":"turn.started"}"#,
1564 "\n",
1565 r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Reviewing..."}}"#,
1566 "\n",
1567 r#"{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the tests\",\"issues\":[]}\n```\n"}}"#,
1568 "\n",
1569 r#"{"type":"turn.completed"}"#,
1570 );
1571 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1572 assert_eq!(resp.decision, "request_changes");
1573 assert_eq!(resp.notes, "Fix the tests");
1574 }
1575
1576 #[serial(cooldown_state)]
1578 #[test]
1579 fn codex_skips_reasoning_items() {
1580 let ndjson = concat!(
1581 r#"{"type":"item.completed","item":{"type":"reasoning","text":"internal thoughts..."}}"#,
1582 "\n",
1583 r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}}"#,
1584 );
1585 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1586 assert_eq!(resp.decision, "approve");
1587 assert_eq!(resp.notes, "LGTM");
1588 }
1589
1590 #[serial(cooldown_state)]
1592 #[test]
1593 fn infer_review_response_negation_not_approved() {
1594 assert!(
1595 parse_review_plain("The changes were not approved.").is_err(),
1596 "\"not approved\" should not infer approval"
1597 );
1598 }
1599
1600 #[serial(cooldown_state)]
1601 #[test]
1602 fn infer_review_response_negation_unapproved() {
1603 assert!(
1604 parse_review_plain("Unapproved changes found in the diff.").is_err(),
1605 "\"unapproved\" should not infer approval"
1606 );
1607 }
1608
1609 #[serial(cooldown_state)]
1610 #[test]
1611 fn infer_review_response_negation_not_approving() {
1612 assert!(
1613 parse_review_plain("I am not approving this PR.").is_err(),
1614 "\"not approving\" should not infer approval"
1615 );
1616 }
1617
1618 #[serial(cooldown_state)]
1619 #[test]
1620 fn infer_review_response_negation_cannot_be_approved() {
1621 assert!(
1622 parse_review_plain("This PR cannot be approved until the race in tick.rs is fixed.")
1623 .is_err(),
1624 "\"cannot be approved\" should not infer approval"
1625 );
1626 }
1627
1628 #[serial(cooldown_state)]
1629 #[test]
1630 fn infer_review_response_negation_should_not_be_approved() {
1631 assert!(
1632 parse_review_plain("The diff breaks the build and should not be approved.").is_err(),
1633 "\"should not be approved\" should not infer approval"
1634 );
1635 }
1636
1637 #[serial(cooldown_state)]
1638 #[test]
1639 fn infer_review_response_negation_not_yet_approved() {
1640 assert!(
1641 parse_review_plain("This change is not yet approved.").is_err(),
1642 "\"not yet approved\" should not infer approval"
1643 );
1644 }
1645
1646 #[serial(cooldown_state)]
1647 #[test]
1648 fn infer_review_response_negation_wont_approve() {
1649 assert!(
1650 parse_review_plain("I won't approve this until tests cover the retry path.").is_err(),
1651 "\"won't approve\" should not infer approval"
1652 );
1653 }
1654
1655 #[serial(cooldown_state)]
1656 #[test]
1657 fn infer_review_response_bare_approved() {
1658 assert!(
1659 parse_review_plain("The PR was approved.").is_err(),
1660 "bare \"approved\" should not infer approval"
1661 );
1662 }
1663
1664 #[serial(cooldown_state)]
1666 #[test]
1667 fn infer_review_response_lgtm() {
1668 let resp = parse_review_plain("LGTM, everything looks fine.").unwrap();
1669 assert_eq!(resp.decision, "approve");
1670 }
1671
1672 #[serial(cooldown_state)]
1674 #[test]
1675 fn infer_review_response_looks_good() {
1676 assert!(
1677 parse_review_plain("Looks good to me!").is_err(),
1678 "\"looks good\" should not infer approval"
1679 );
1680 }
1681
1682 #[serial(cooldown_state)]
1683 #[test]
1684 fn infer_review_response_all_checks_passed() {
1685 assert!(
1686 parse_review_plain(
1687 "The background task completed but I already got the CI results directly. All checks passed — no action needed.",
1688 )
1689 .is_err(),
1690 "\"all checks passed\" should not infer approval"
1691 );
1692 }
1693
1694 #[serial(cooldown_state)]
1695 #[test]
1696 fn infer_review_response_checks_passed() {
1697 assert!(
1698 parse_review_plain("All checks passed (fmt ✓, clippy ✓, tests ✓).").is_err(),
1699 "\"all checks passed\" should not infer approval"
1700 );
1701 }
1702
1703 #[serial(cooldown_state)]
1704 #[test]
1705 fn infer_review_response_all_tests_passed() {
1706 assert!(
1707 parse_review_plain("All tests passed and the implementation looks correct.").is_err(),
1708 "\"all tests passed\" should not infer approval"
1709 );
1710 }
1711
1712 #[serial(cooldown_state)]
1713 #[test]
1714 fn infer_review_response_no_issues_found() {
1715 assert!(
1716 parse_review_plain("Review complete. No issues found in the diff.").is_err(),
1717 "\"no issues found\" should not infer approval"
1718 );
1719 }
1720
1721 #[serial(cooldown_state)]
1723 #[test]
1724 fn infer_review_response_partial_tests_passed_no_approve() {
1725 let text = "The existing tests passed before this change but the new payment logic has a null-pointer risk on line 42.";
1726 assert!(parse_review_plain(text).is_err());
1727 }
1728
1729 #[serial(cooldown_state)]
1731 #[test]
1732 fn infer_review_response_partial_checks_passed_no_approve() {
1733 let text = "CI checks passed for the base branch but this PR adds a broken path.";
1734 assert!(parse_review_plain(text).is_err());
1735 }
1736
1737 #[serial(cooldown_state)]
1739 #[test]
1740 fn infer_review_response_decision_is_approve_backtick() {
1741 let text = "Already retrieved the output and completed the review above. The CI test check passed (3m11s) and all other checks are green. Decision is `approve`.";
1742 let resp = parse_review_plain(text).unwrap();
1743 assert_eq!(resp.decision, "approve");
1744 }
1745
1746 #[serial(cooldown_state)]
1748 #[test]
1749 fn infer_review_response_decision_colon_approve() {
1750 let text = "Reviewed the PR. No blocking issues found. Decision: approve";
1751 let resp = parse_review_plain(text).unwrap();
1752 assert_eq!(resp.decision, "approve");
1753 }
1754
1755 #[serial(cooldown_state)]
1758 #[test]
1759 fn infer_review_response_negation_do_not_approve_backtick() {
1760 let text = "There are unresolved issues — do not `approve` this PR until they are fixed.";
1761 assert!(
1762 parse_review_plain(text).is_err(),
1763 "negative context with backtick approve must not infer approval"
1764 );
1765 }
1766
1767 fn parse_via_agent_path(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
1772 let text = find_agent_result(agent, output)
1773 .map(|r| r.result_text)
1774 .filter(|t| !t.is_empty())
1775 .unwrap_or_else(|| output.to_string());
1776 if let Ok(r) = parse_review_response(&text) {
1777 return Ok(r);
1778 }
1779 infer_review_response_from_text(&text)
1780 .ok_or_else(|| anyhow::anyhow!("failed to parse review response from {agent} output"))
1781 }
1782
1783 #[serial(cooldown_state)]
1784 #[test]
1785 fn agent_output_claude_ndjson_review() {
1786 let ndjson = concat!(
1787 r#"{"type":"system","subtype":"init","session_id":"s1"}"#,
1788 "\n",
1789 r#"{"type":"assistant","message":{"content":[{"type":"text","text":"Reviewing..."}]}}"#,
1790 "\n",
1791 r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}","usage":{"input_tokens":100,"output_tokens":20}}"#,
1792 );
1793 let resp = parse_via_agent_path("claude", ndjson).unwrap();
1794 assert_eq!(resp.decision, "approve");
1795 }
1796
1797 #[serial(cooldown_state)]
1798 #[test]
1799 fn agent_output_opencode_ndjson_review() {
1800 let ndjson = concat!(
1801 r#"{"type":"step_start","timestamp":1000}"#,
1802 "\n",
1803 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}"}}"#,
1804 "\n",
1805 r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop","tokens":{"input":100,"output":10}}}"#,
1806 );
1807 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1808 assert_eq!(resp.decision, "request_changes");
1809 }
1810
1811 #[serial(cooldown_state)]
1812 #[test]
1813 fn agent_output_codex_ndjson_review() {
1814 let ndjson = concat!(
1815 r#"{"type":"thread.started","thread_id":"t1"}"#,
1816 "\n",
1817 r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All good\",\"issues\":[]}"}}"#,
1818 "\n",
1819 r#"{"type":"turn.completed"}"#,
1820 );
1821 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1822 assert_eq!(resp.decision, "approve");
1823 }
1824
1825 #[serial(cooldown_state)]
1826 #[test]
1827 fn agent_output_plain_text_keyword_fallback() {
1828 let text = "All checks passed, LGTM.";
1829 let resp = parse_via_agent_path("claude", text).unwrap();
1830 assert_eq!(resp.decision, "approve");
1831 }
1832
1833 #[serial(cooldown_state)]
1834 #[test]
1835 fn agent_output_direct_json() {
1836 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1837 let resp = parse_via_agent_path("opencode", json).unwrap();
1838 assert_eq!(resp.decision, "approve");
1839 }
1840
1841 #[serial(cooldown_state)]
1845 #[test]
1846 fn agent_output_text_with_json_fragment_no_false_positive() {
1847 let text = r#"I checked the file and found {"type":"text"} in the output. No issues."#;
1848 let result = parse_via_agent_path("claude", text);
1852 assert!(
1854 result.is_err() || result.as_ref().is_ok_and(|r| r.decision != "text"),
1855 "should not extract review from embedded JSON fragment"
1856 );
1857 }
1858
1859 #[serial(cooldown_state)]
1862 #[test]
1863 fn parse_review_response_ignores_prose_with_decision_word() {
1864 let text = r#"The decision was made. Here is a debug trace: {"trace":123}"#;
1865 assert!(
1866 parse_review_response(text).is_err(),
1867 "prose with 'decision' word and unrelated JSON should not parse as ReviewResponse"
1868 );
1869 }
1870
1871 #[serial(cooldown_state)]
1874 #[test]
1875 fn parse_review_response_ignores_embedded_json_without_decision_key() {
1876 let text = r#"Analysis complete. Config: {"timeout":30,"retries":3}. Done."#;
1877 assert!(
1878 parse_review_response(text).is_err(),
1879 "embedded JSON without 'decision' key should not parse as ReviewResponse"
1880 );
1881 }
1882
1883 #[serial(cooldown_state)]
1886 #[test]
1887 fn read_file_with_limit_reads_small_file_completely() {
1888 let temp_dir = env::temp_dir();
1889 let file_path = temp_dir.join(format!("orch_test_small_{}.txt", process::id()));
1890 let content = "Hello, World!";
1891 fs::write(&file_path, content).unwrap();
1892
1893 let result = read_file_with_limit(&file_path, 1024);
1894 assert!(result.is_ok());
1895 assert_eq!(result.unwrap(), content);
1896
1897 let _ = fs::remove_file(&file_path);
1898 }
1899
1900 #[serial(cooldown_state)]
1901 #[test]
1902 fn read_file_with_limit_truncates_large_file() {
1903 let temp_dir = env::temp_dir();
1904 let file_path = temp_dir.join(format!("orch_test_large_{}.txt", process::id()));
1905
1906 let large_content = "x".repeat(200);
1908 fs::write(&file_path, &large_content).unwrap();
1909
1910 let result = read_file_with_limit(&file_path, 50);
1912 assert!(result.is_ok());
1913
1914 let read_content = result.unwrap();
1915 assert_eq!(read_content.len(), 50);
1917 assert_eq!(read_content, "x".repeat(50));
1918
1919 let _ = fs::remove_file(&file_path);
1920 }
1921
1922 #[serial(cooldown_state)]
1923 #[test]
1924 fn read_file_with_limit_handles_empty_file() {
1925 let temp_dir = env::temp_dir();
1926 let file_path = temp_dir.join(format!("orch_test_empty_{}.txt", process::id()));
1927 fs::write(&file_path, "").unwrap();
1928
1929 let result = read_file_with_limit(&file_path, 1024);
1930 assert!(result.is_ok());
1931 assert_eq!(result.unwrap(), "");
1932
1933 let _ = fs::remove_file(&file_path);
1934 }
1935
1936 #[serial(cooldown_state)]
1937 #[test]
1938 fn read_file_with_limit_handles_nonexistent_file() {
1939 let temp_dir = env::temp_dir();
1940 let file_path = temp_dir.join("orch_test_nonexistent_xyz123.txt");
1941
1942 let result = read_file_with_limit(&file_path, 1024);
1943 assert!(result.is_err());
1944 }
1945
1946 #[serial(cooldown_state)]
1947 #[test]
1948 fn read_file_with_limit_reads_exact_limit() {
1949 let temp_dir = env::temp_dir();
1950 let file_path = temp_dir.join(format!("orch_test_exact_{}.txt", process::id()));
1951
1952 let content = "y".repeat(100);
1954 fs::write(&file_path, &content).unwrap();
1955
1956 let result = read_file_with_limit(&file_path, 100);
1958 assert!(result.is_ok());
1959 assert_eq!(result.unwrap().len(), 100);
1960
1961 let _ = fs::remove_file(&file_path);
1962 }
1963
1964 #[serial(cooldown_state)]
1965 #[test]
1966 fn read_file_with_limit_handles_unicode_truncation() {
1967 let temp_dir = env::temp_dir();
1968 let file_path = temp_dir.join(format!("orch_test_unicode_{}.txt", process::id()));
1969
1970 let emoji = "🎉".repeat(100); fs::write(&file_path, &emoji).unwrap();
1973
1974 let result = read_file_with_limit(&file_path, 50);
1976 assert!(result.is_ok());
1977
1978 let read_content = result.unwrap();
1979 assert_ne!(read_content, "");
1981
1982 let _ = fs::remove_file(&file_path);
1983 }
1984
1985 fn parse_review_via_agent(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
1989 let text = find_agent_result(agent, output)
1990 .map(|r| r.result_text)
1991 .filter(|t| !t.is_empty())
1992 .unwrap_or_else(|| output.to_string());
1993 if let Ok(r) = parse_review_response(&text) {
1994 return Ok(r);
1995 }
1996 infer_review_response(&text)
1997 .ok_or_else(|| anyhow::anyhow!("failed to parse review from {agent} output"))
1998 }
1999
2000 #[serial(cooldown_state)]
2003 #[test]
2004 fn fixture_claude_review_approve() {
2005 let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2006 let resp = parse_review_via_agent("claude", raw).unwrap();
2007 assert_eq!(resp.decision, "approve");
2008 assert_eq!(resp.notes, "All checks pass");
2009 assert_eq!(resp.test_results.as_deref(), Some("pass"));
2010 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2011 }
2012
2013 #[serial(cooldown_state)]
2014 #[test]
2015 fn fixture_claude_review_request_changes() {
2016 let raw = include_str!("../../../tests/fixtures/review_claude_request_changes.jsonl");
2017 let resp = parse_review_via_agent("claude", raw).unwrap();
2018 assert_eq!(resp.decision, "request_changes");
2019 assert_eq!(resp.notes, "Fix the null deref on line 42");
2020 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2021 assert_eq!(resp.issues.len(), 1);
2022 assert_eq!(resp.issues[0].file, "src/main.rs");
2023 assert_eq!(resp.issues[0].line, Some(42));
2024 assert_eq!(resp.issues[0].severity, "error");
2025 }
2026
2027 #[serial(cooldown_state)]
2028 #[test]
2029 fn fixture_claude_review_rate_limit_returns_error() {
2030 let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2033 let result = find_agent_result("claude", raw);
2034 assert!(
2035 result.is_some(),
2036 "should find result even for error envelope"
2037 );
2038 let result = result.unwrap();
2039 assert!(result.is_error, "rate limit should be flagged as error");
2040 assert!(result.result_text.contains("rate limit"));
2041 }
2042
2043 #[serial(cooldown_state)]
2046 #[test]
2047 fn fixture_opencode_review_approve() {
2048 let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2049 let resp = parse_review_via_agent("opencode", raw).unwrap();
2050 assert_eq!(resp.decision, "approve");
2051 assert_eq!(resp.notes, "All checks pass");
2052 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2053 }
2054
2055 #[serial(cooldown_state)]
2056 #[test]
2057 fn fixture_opencode_review_request_changes() {
2058 let raw = include_str!("../../../tests/fixtures/review_opencode_request_changes.jsonl");
2059 let resp = parse_review_via_agent("opencode", raw).unwrap();
2060 assert_eq!(resp.decision, "request_changes");
2061 assert_eq!(resp.notes, "Fix the memory leak");
2062 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2063 assert_eq!(resp.issues.len(), 1);
2064 assert_eq!(resp.issues[0].file, "src/lib.rs");
2065 assert_eq!(resp.issues[0].line, Some(100));
2066 }
2067
2068 #[serial(cooldown_state)]
2069 #[test]
2070 fn fixture_opencode_review_plain_text_approve() {
2071 let raw = include_str!("../../../tests/fixtures/review_opencode_plain_text.jsonl");
2073 let resp = parse_review_via_agent("opencode", raw).unwrap();
2074 assert_eq!(resp.decision, "approve");
2075 }
2076
2077 #[serial(cooldown_state)]
2080 #[test]
2081 fn fixture_codex_review_approve() {
2082 let raw = include_str!("../../../tests/fixtures/review_codex_approve.jsonl");
2083 let resp = parse_review_via_agent("codex", raw).unwrap();
2084 assert_eq!(resp.decision, "approve");
2085 assert_eq!(resp.notes, "Clean implementation");
2086 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2087 }
2088
2089 #[serial(cooldown_state)]
2090 #[test]
2091 fn fixture_codex_review_request_changes() {
2092 let raw = include_str!("../../../tests/fixtures/review_codex_request_changes.jsonl");
2093 let resp = parse_review_via_agent("codex", raw).unwrap();
2094 assert_eq!(resp.decision, "request_changes");
2095 assert_eq!(resp.notes, "Race condition on line 15");
2096 assert_eq!(resp.issues.len(), 1);
2097 assert_eq!(resp.issues[0].file, "src/server.rs");
2098 assert_eq!(resp.issues[0].line, Some(15));
2099 }
2100
2101 #[serial(cooldown_state)]
2102 #[test]
2103 fn fixture_codex_review_plain_text_approve() {
2104 let raw = include_str!("../../../tests/fixtures/review_codex_plain_text.jsonl");
2106 let resp = parse_review_via_agent("codex", raw).unwrap();
2107 assert_eq!(resp.decision, "approve");
2108 }
2109
2110 #[serial(cooldown_state)]
2113 #[test]
2114 fn fixture_kimi_review_approve() {
2115 let raw = include_str!("../../../tests/fixtures/review_kimi_approve.jsonl");
2116 let resp = parse_review_via_agent("kimi", raw).unwrap();
2117 assert_eq!(resp.decision, "approve");
2118 assert_eq!(resp.notes, "LGTM");
2119 assert_eq!(resp.test_results.as_deref(), Some("pass"));
2120 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2121 }
2122
2123 #[serial(cooldown_state)]
2126 #[test]
2127 fn fixture_minimax_review_request_changes() {
2128 let raw = include_str!("../../../tests/fixtures/review_minimax_request_changes.jsonl");
2129 let resp = parse_review_via_agent("minimax", raw).unwrap();
2130 assert_eq!(resp.decision, "request_changes");
2131 assert_eq!(resp.notes, "Fix the bug");
2132 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2133 assert_eq!(resp.issues.len(), 1);
2134 assert_eq!(resp.issues[0].file, "src/foo.rs");
2135 assert_eq!(resp.issues[0].line, Some(42));
2136 assert_eq!(resp.issues[0].severity, "error");
2137 }
2138
2139 #[serial(cooldown_state)]
2143 #[test]
2144 fn find_agent_result_extracts_tokens_claude() {
2145 let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2146 let result = find_agent_result("claude", raw).expect("should find result");
2147 assert_eq!(result.input_tokens, Some(200));
2148 assert_eq!(result.output_tokens, Some(30));
2149 assert!(result.cost_usd.is_none()); }
2151
2152 #[serial(cooldown_state)]
2153 #[test]
2154 fn find_agent_result_extracts_tokens_opencode() {
2155 let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2156 let result = find_agent_result("opencode", raw).expect("should find result");
2157 assert_eq!(result.input_tokens, Some(2900));
2158 assert_eq!(result.output_tokens, Some(100));
2159 assert_eq!(result.cost_usd, Some(0.002));
2160 }
2161
2162 #[serial(cooldown_state)]
2164 #[test]
2165 fn find_agent_result_returns_none_for_empty() {
2166 assert!(find_agent_result("claude", "").is_none());
2167 assert!(find_agent_result("opencode", "").is_none());
2168 assert!(find_agent_result("codex", "").is_none());
2169 assert!(find_agent_result("kimi", "").is_none());
2170 assert!(find_agent_result("minimax", "").is_none());
2171 }
2172
2173 #[serial(cooldown_state)]
2174 #[test]
2175 fn find_agent_result_returns_none_for_plain_text() {
2176 assert!(find_agent_result("claude", "just some plain text").is_none());
2177 assert!(find_agent_result("codex", "just some plain text").is_none());
2178 }
2181
2182 #[serial(cooldown_state)]
2186 #[test]
2187 fn find_agent_result_propagates_claude_rate_limit() {
2188 let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2189 let result = find_agent_result("claude", raw).expect("should find result");
2190 assert!(result.is_error, "should be flagged as error");
2191 assert!(result.result_text.contains("rate limit"));
2192 }
2193
2194 #[cfg(unix)]
2197 fn failover_env() -> tempfile::TempDir {
2198 let dir = tempfile::tempdir().unwrap();
2199 let bin = dir.path().join("bin");
2200 fs::create_dir(&bin).unwrap();
2201 for agent in ["claude", "codex"] {
2202 let path = bin.join(agent);
2203 fs::write(&path, "#!/bin/sh\n").unwrap();
2204 fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap();
2205 }
2206 let home = dir.path().join("home");
2207 fs::create_dir(&home).unwrap();
2208 fs::write(
2209 home.join("config.yml"),
2210 "agents: [claude, codex]\ndispatch:\n retry_base_delay_ms: 1\n retry_max_delay_ms: 1\nworkflow:\n max_timeout_retries: 3\n",
2211 )
2212 .unwrap();
2213 let mut path = bin.as_os_str().to_os_string();
2214 path.push(":");
2215 path.push(env::var_os("PATH").unwrap_or_default());
2216 unsafe { env::set_var("PATH", path) };
2218 unsafe { env::set_var("ORCH_HOME", &home) };
2220 cmd_cache::reset_for_test();
2221 dir
2222 }
2223
2224 async fn failover_task() -> (Arc<TaskStore>, String) {
2225 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
2226 let id = store
2227 .create_internal("owner/repo", "t", "b", "test", "", None)
2228 .await
2229 .unwrap();
2230 (store, format!("internal:{id}"))
2231 }
2232
2233 #[cfg(unix)]
2235 #[serial(cooldown_state)]
2236 #[tokio::test]
2237 async fn timeout_with_every_other_agent_cooled_retries_as_new() {
2238 let _env = failover_env();
2239 cooldown::reset_global_state().await;
2240 set_agent_cooldown("codex", 600).await;
2241 let (store, task_id) = failover_task().await;
2242 let store = Some(store);
2243
2244 let status = handle_failover(
2245 &task_id,
2246 "claude",
2247 RetryableError::Timeout,
2248 "claude timed out after 1804s",
2249 false,
2250 &store,
2251 "owner/repo",
2252 )
2253 .await;
2254
2255 assert_eq!(status, "new");
2256 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2257 .await
2258 .unwrap();
2259 assert!(task.limit_reroute_chain.is_empty(), "chain must be cleared");
2260 assert!(
2261 task.last_error.contains("will retry"),
2262 "{}",
2263 task.last_error
2264 );
2265 assert_eq!(task.timeout_retries, 1);
2266 }
2267
2268 #[cfg(unix)]
2270 #[serial(cooldown_state)]
2271 #[tokio::test]
2272 async fn timeout_with_a_free_agent_reroutes_to_it() {
2273 let _env = failover_env();
2274 cooldown::reset_global_state().await;
2275 let (store, task_id) = failover_task().await;
2276 let store = Some(store);
2277
2278 let status = handle_failover(
2279 &task_id,
2280 "claude",
2281 RetryableError::Timeout,
2282 "claude timed out after 1804s",
2283 false,
2284 &store,
2285 "owner/repo",
2286 )
2287 .await;
2288
2289 assert_eq!(status, "new");
2290 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2291 .await
2292 .unwrap();
2293 assert_eq!(task.limit_reroute_chain, "claude");
2294 assert!(task.last_error.contains("re-route"), "{}", task.last_error);
2295 }
2296
2297 #[cfg(unix)]
2299 #[serial(cooldown_state)]
2300 #[tokio::test]
2301 async fn repeated_timeouts_block_the_task() {
2302 let _env = failover_env();
2303 cooldown::reset_global_state().await;
2304 set_agent_cooldown("codex", 600).await;
2305 let (store, task_id) = failover_task().await;
2306 let store = Some(store);
2307
2308 let mut statuses = Vec::new();
2309 for _ in 0..3 {
2310 statuses.push(
2311 handle_failover(
2312 &task_id,
2313 "claude",
2314 RetryableError::Timeout,
2315 "claude timed out after 1804s",
2316 false,
2317 &store,
2318 "owner/repo",
2319 )
2320 .await,
2321 );
2322 }
2323
2324 assert_eq!(statuses, ["new", "new", "blocked"]);
2325 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2326 .await
2327 .unwrap();
2328 assert_eq!(task.block_reason.as_deref(), Some("timed out 3 times"));
2329 assert_eq!(task.timeout_retries, 3);
2330 }
2331}