1use crate::store;
10use crate::store::TaskStore;
11use std::collections::HashSet;
12use std::collections::hash_map::DefaultHasher;
13use std::fs;
14use std::fs::File;
15use std::hash::Hash;
16use std::hash::Hasher;
17use std::io::Read;
18use std::io::Seek;
19use std::io::SeekFrom;
20use std::path::{Path, PathBuf};
21use std::result;
22use std::sync::Arc;
23use std::thread;
24use std::time::Duration;
25use std::time::SystemTime;
26
27use tokio::fs as tokio_fs;
28use tokio::task;
29use tokio::time;
30use crate::cli::ndjson;
32use crate::cmd_cache;
33use crate::config as app_config;
34use crate::engine;
35pub use crate::engine::cooldown::{
36 clear_expired_cooldowns, is_agent_in_cooldown, is_model_in_cooldown,
37 record_agent_failure_with_message, record_model_failure, set_agent_cooldown,
38};
39use crate::engine::router::config;
40use crate::home;
41use crate::parser::AgentResponse;
42use crate::store::MemoryEntry;
43use crate::store::Task;
44
45const JITTER_FACTOR: f64 = 0.3;
46
47const MAX_OUTPUT_SIZE_BYTES: u64 = 10 * 1024 * 1024;
50
51fn read_file_with_limit<P: AsRef<Path>>(path: P, max_bytes: u64) -> anyhow::Result<String> {
57 let path = path.as_ref();
58 let metadata = fs::metadata(path)?;
59
60 if !metadata.is_file() {
61 anyhow::bail!("not a file: {}", path.display());
62 }
63
64 let file_size = metadata.len();
65
66 if file_size <= max_bytes {
68 return fs::read_to_string(path).map_err(Into::into);
69 }
70
71 let mut file = File::open(path)?;
73 let start_pos = file_size.saturating_sub(max_bytes);
74 file.seek(SeekFrom::Start(start_pos))?;
75
76 let mut buffer = Vec::with_capacity(usize::try_from(max_bytes).unwrap_or(0));
77 file.read_to_end(&mut buffer)?;
78
79 let content = String::from_utf8_lossy(&buffer);
81
82 tracing::warn!(
84 path = %path.display(),
85 file_size = file_size,
86 max_bytes = max_bytes,
87 "output file truncated: exceeded size limit"
88 );
89
90 Ok(content.into_owned())
91}
92
93pub(crate) fn calculate_backoff_delay(retry_count: usize) -> u64 {
98 let base_delay = config::retry_base_delay_ms();
99 let max_delay = config::retry_max_delay_ms();
100
101 let exponential =
103 base_delay * (2_u64.saturating_pow(u32::try_from(retry_count).unwrap_or(u32::MAX)));
104 let capped = exponential.min(max_delay);
105
106 #[expect(
111 clippy::cast_precision_loss,
112 clippy::cast_possible_truncation,
113 clippy::cast_sign_loss,
114 reason = "a jitter range in milliseconds, far below 2^52 and never negative"
115 )]
116 let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
117 let jitter = {
118 let mut h = DefaultHasher::new();
119 SystemTime::now().hash(&mut h);
120 thread::current().id().hash(&mut h);
121 h.finish() % (jitter_range * 2 + 1)
122 };
123
124 capped.saturating_sub(jitter_range) + jitter
125}
126
127fn fallback_retry_count(task: Option<&Task>) -> usize {
128 task.map_or(0, |t| {
129 let chain_retry_count = t
130 .limit_reroute_chain
131 .split(',')
132 .filter(|entry| !entry.is_empty())
133 .count();
134 let network_retry_count = usize::try_from(t.network_retries).unwrap_or(0);
135 chain_retry_count.max(network_retry_count)
136 })
137}
138
139pub async fn wait_for_fallback_backoff(
142 task_id: &str,
143 store: &Option<Arc<TaskStore>>,
144 repo: &str,
145) -> u64 {
146 let task = store::opt_store_get_task(store, repo, task_id).await;
147
148 let retry_count = fallback_retry_count(task.as_ref());
151
152 let delay = calculate_backoff_delay(retry_count);
153
154 if delay > 0 {
155 tracing::debug!(
156 task_id,
157 retry_count = retry_count,
158 delay_ms = delay,
159 "fallback backoff: waiting before retry"
160 );
161 time::sleep(Duration::from_millis(delay)).await;
162 }
163
164 delay
165}
166
167#[derive(Debug, Clone)]
172pub enum WeightSignal {
173 Success { agent: String },
175 RateLimited { agent: String },
177 Failed { agent: String },
180 Blocked,
182 Rerouted,
185 None,
187}
188
189pub async fn read_output_file(task_id: &str, primary_path: &Path, repo: &str) -> String {
194 if let Some(content) = task::spawn_blocking({
196 let p = primary_path.to_path_buf();
197 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
198 })
199 .await
200 .ok()
201 .and_then(result::Result::ok)
202 && !content.is_empty()
203 {
204 return content;
205 }
206
207 if let Ok(task_dir) = home::task_dir_async(repo, task_id).await {
209 let attempts_dir = task_dir.join("attempts");
210 if tokio_fs::metadata(&attempts_dir)
212 .await
213 .is_ok_and(|m| m.is_dir())
214 {
215 let mut attempt_nums: Vec<u32> = task::spawn_blocking({
216 let a = attempts_dir.clone();
217 move || {
218 match fs::read_dir(&a) {
219 Ok(entries) => {
220 let mut nums = Vec::new();
221 for entry in entries {
222 match entry {
223 Ok(e) => {
224 if let Some(n) =
225 e.file_name().to_str().and_then(|n| n.parse().ok())
226 {
227 nums.push(n);
228 }
229 }
230 Err(e) => tracing::warn!(
231 path = %a.display(),
232 err = %e,
233 "error reading attempt dir entry in output fallback"
234 ),
235 }
236 }
237 nums
238 }
239 Err(e) => {
240 tracing::warn!(path = %a.display(), err = %e, "failed to read attempts dir in output fallback");
241 Vec::new()
242 }
243 }
244 }
245 })
246 .await
247 .unwrap_or_default();
248
249 attempt_nums.sort_unstable_by(|a, b| b.cmp(a)); for n in attempt_nums {
251 let p = attempts_dir.join(n.to_string()).join("output.json");
252 if let Some(content) = task::spawn_blocking({
253 let p = p.clone();
254 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
255 })
256 .await
257 .ok()
258 .and_then(result::Result::ok)
259 && !content.is_empty()
260 {
261 tracing::info!(task_id, path = %p.display(), "read output from attempt dir");
262 return content;
263 }
264 }
265 }
266 }
267
268 let state_dir = home::state_dir().unwrap_or_else(|_| PathBuf::from("/tmp"));
270
271 let mut fallbacks = vec![
272 PathBuf::from(format!("/tmp/output-{task_id}.json")),
273 state_dir.join(format!("output-{task_id}.json")),
274 ];
275
276 if let Ok(legacy_path) = home::state_file(&format!("output-{task_id}.json"))
277 && !fallbacks.contains(&legacy_path)
278 {
279 fallbacks.push(legacy_path);
280 }
281
282 for path in &fallbacks {
283 if let Some(content) = task::spawn_blocking({
284 let p = path.clone();
285 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
286 })
287 .await
288 .ok()
289 .and_then(result::Result::ok)
290 && !content.is_empty()
291 {
292 tracing::info!(task_id, path = %path.display(), "read output from legacy fallback");
293 return content;
294 }
295 }
296
297 String::new()
298}
299
300pub fn pick_fallback_agent(
302 current_agent: &str,
303 chain: &str,
304 available_agents: &[String],
305) -> Option<String> {
306 clear_expired_cooldowns();
308
309 let chain_set: HashSet<&str> = if chain.is_empty() {
310 HashSet::new()
311 } else {
312 chain.split(',').collect()
313 };
314
315 for agent in available_agents {
319 if agent == current_agent || chain_set.contains(agent.as_str()) {
320 continue;
321 }
322 if is_agent_in_cooldown(agent) {
323 continue;
324 }
325 return Some(agent.clone());
332 }
333
334 None
335}
336
337#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum RetryableError {
340 Timeout,
342 Interrupted,
344 UsageLimit,
346 AuthError,
348 Failed,
350 MissingTooling,
352 ModelUnavailable,
356}
357
358impl RetryableError {
359 pub fn type_str(self) -> &'static str {
362 match self {
363 RetryableError::Timeout => "timeout",
364 RetryableError::Interrupted => "interrupted",
365 RetryableError::UsageLimit => "rate_limit",
366 RetryableError::AuthError => "auth_error",
367 RetryableError::ModelUnavailable => "model_unavailable",
368 RetryableError::Failed | RetryableError::MissingTooling => "failed",
369 }
370 }
371}
372
373#[derive(Debug, Clone, PartialEq, Eq)]
375pub struct FailoverOutcome {
376 pub status: String,
378 pub next_agent: Option<String>,
380}
381
382impl FailoverOutcome {
383 fn with_status(status: impl Into<String>) -> Self {
384 Self {
385 status: status.into(),
386 next_agent: None,
387 }
388 }
389
390 fn switched(next_agent: String) -> Self {
391 Self {
392 status: "new".to_string(),
393 next_agent: Some(next_agent),
394 }
395 }
396}
397
398pub async fn handle_failover(
409 task_id: &str,
410 agent_name: &str,
411 error_type: RetryableError,
412 error_message: &str,
413 credit_recorded: bool,
414 store: &Option<Arc<TaskStore>>,
415 repo: &str,
416) -> FailoverOutcome {
417 let chain = get_reroute_chain(task_id, store, repo).await;
419 let chain = update_reroute_chain(task_id, agent_name, &chain, store, repo).await;
420
421 if error_type == RetryableError::Timeout
422 && let Some(status) =
423 block_after_repeated_timeouts(task_id, error_message, store, repo).await
424 {
425 return FailoverOutcome::with_status(status);
426 }
427
428 let available: Vec<String> = engine::configured_agents()
430 .into_iter()
431 .filter(|a| cmd_cache::command_exists(a))
432 .collect();
433
434 let chain_set: HashSet<&str> = if chain.is_empty() {
436 HashSet::new()
437 } else {
438 chain.split(',').collect()
439 };
440 let is_exhausted = available.iter().all(|a| chain_set.contains(a.as_str()));
441
442 if is_exhausted {
443 let is_retryable = matches!(
449 error_type,
450 RetryableError::UsageLimit
451 | RetryableError::AuthError
452 | RetryableError::ModelUnavailable
453 | RetryableError::Timeout
454 | RetryableError::Interrupted
455 );
456 let next_status = if is_retryable { "new" } else { "blocked" };
457
458 tracing::warn!(
459 task_id,
460 agent = agent_name,
461 error_type = ?error_type,
462 next_status,
463 "all agents exhausted"
464 );
465
466 let mut store_fields: Vec<(&str, serde_json::Value)> = Vec::new();
467
468 if is_retryable {
469 let msg = format!("{error_message} (all agents exhausted, will retry when available)");
472 store_fields.push(("last_error", serde_json::json!(msg)));
473 store_fields.push(("limit_reroute_chain", serde_json::json!("")));
474 } else {
475 let msg = format!("{error_message} (all agents exhausted)");
476 let block_reason = format!(
477 "all agents exhausted: {} — {}",
478 error_type.type_str(),
479 error_message
480 );
481 store_fields.push(("last_error", serde_json::json!(msg)));
482 store_fields.push(("block_reason", serde_json::json!(block_reason)));
483 }
484
485 if let Err(e) = store::store_set_result(store, repo, task_id, &store_fields).await {
486 tracing::warn!(task_id, err = %e, "failed to write agents_exhausted fields to store");
487 }
488
489 let details = serde_json::json!({
491 "failure_reason": error_type.type_str(),
492 "error_message": error_message,
493 "chain": chain,
494 "agents_exhausted": true,
495 "is_retryable": is_retryable,
496 });
497 store::store_log_activity(
498 store,
499 repo,
500 task_id,
501 "rerouted",
502 Some("in_progress"),
503 Some(next_status),
504 None::<&str>,
505 None::<&str>,
506 Some(&details),
507 )
508 .await;
509
510 return FailoverOutcome::with_status(next_status);
511 }
512
513 if let Some(next) = pick_fallback_agent(agent_name, &chain, &available) {
515 tracing::info!(
516 task_id,
517 from = agent_name,
518 to = next,
519 error_type = ?error_type,
520 "failover: switching to fallback agent"
521 );
522
523 if !matches!(
525 error_type,
526 RetryableError::MissingTooling
527 | RetryableError::ModelUnavailable
528 | RetryableError::Timeout
529 | RetryableError::Interrupted
530 ) {
531 if !credit_recorded {
534 record_agent_failure_with_message(agent_name, error_message).await;
535 }
536
537 set_agent_cooldown(agent_name, 120).await;
542 }
543
544 let msg = format!("{error_message}, switching to {next} for re-route");
547 if let Err(e) = store::store_set_result(
548 store,
549 repo,
550 task_id,
551 &[
552 ("agent", serde_json::json!(next)),
553 ("model", serde_json::json!("")),
554 ("last_error", serde_json::json!(msg)),
555 ],
556 )
557 .await
558 {
559 tracing::warn!(task_id, err = %e, "failed to write failover agent to store");
560 }
561
562 wait_for_fallback_backoff(task_id, store, repo).await;
568
569 let details = serde_json::json!({
571 "failure_reason": error_type.type_str(),
572 "error_message": error_message,
573 "from_agent": agent_name,
574 "to_agent": next,
575 "chain": chain,
576 });
577 store::store_log_activity(
578 store,
579 repo,
580 task_id,
581 "rerouted",
582 Some("in_progress"),
583 Some("new"),
584 Some(&next),
585 None::<&str>,
586 Some(&details),
587 )
588 .await;
589
590 return FailoverOutcome::switched(next);
591 }
592
593 tracing::warn!(task_id, agent = agent_name, "no fallback agents available");
595
596 let is_retryable = matches!(
597 error_type,
598 RetryableError::UsageLimit
599 | RetryableError::AuthError
600 | RetryableError::ModelUnavailable
601 | RetryableError::Timeout
602 | RetryableError::Interrupted
603 );
604
605 if !matches!(
607 error_type,
608 RetryableError::MissingTooling
609 | RetryableError::ModelUnavailable
610 | RetryableError::Timeout
611 | RetryableError::Interrupted
612 ) && !credit_recorded
613 {
614 record_agent_failure_with_message(agent_name, error_message).await;
615 }
616
617 if is_retryable {
618 wait_for_fallback_backoff(task_id, store, repo).await;
621
622 let msg = format!("{error_message} (no fallback agents, will retry)");
623 if let Err(e) = store::store_set_result(
624 store,
625 repo,
626 task_id,
627 &[
628 ("last_error", serde_json::json!(msg)),
629 ("limit_reroute_chain", serde_json::json!("")),
630 ],
631 )
632 .await
633 {
634 tracing::warn!(task_id, err = %e, "failed to write no_fallback retry fields to store");
635 }
636
637 let details = serde_json::json!({
638 "failure_reason": error_type.type_str(),
639 "error_message": error_message,
640 "chain": chain,
641 "no_fallback_available": true,
642 "is_retryable": true,
643 });
644 store::store_log_activity(
645 store,
646 repo,
647 task_id,
648 "rerouted",
649 Some("in_progress"),
650 Some("new"),
651 None::<&str>,
652 None::<&str>,
653 Some(&details),
654 )
655 .await;
656
657 return FailoverOutcome::with_status("new");
658 }
659
660 let msg = format!("{error_message}, no fallback agents");
661 if let Err(e) = store::store_set_result(
662 store,
663 repo,
664 task_id,
665 &[("last_error", serde_json::json!(msg))],
666 )
667 .await
668 {
669 tracing::warn!(task_id, err = %e, "failed to write no_fallback last_error to store");
670 }
671
672 let details = serde_json::json!({
674 "failure_reason": error_type.type_str(),
675 "error_message": error_message,
676 "chain": chain,
677 "no_fallback_available": true,
678 });
679 store::store_log_activity(
680 store,
681 repo,
682 task_id,
683 "rerouted",
684 Some("in_progress"),
685 Some("needs_review"),
686 None::<&str>,
687 None::<&str>,
688 Some(&details),
689 )
690 .await;
691
692 FailoverOutcome::with_status("needs_review")
693}
694
695pub async fn block_after_repeated_timeouts(
698 task_id: &str,
699 error_message: &str,
700 store: &Option<Arc<TaskStore>>,
701 repo: &str,
702) -> Option<String> {
703 let max: i64 = app_config::get("workflow.max_timeout_retries")
704 .ok()
705 .and_then(|v| v.parse().ok())
706 .unwrap_or(3);
707 let count = match store::store_increment(store, repo, task_id, "timeout_retries").await {
708 Ok(count) => i64::try_from(count).unwrap_or(i64::MAX),
709 Err(e) => {
710 tracing::warn!(task_id, err = %e, "failed to increment timeout_retries");
711 return None;
712 }
713 };
714 if count < max {
715 return None;
716 }
717
718 let block_reason = format!("timed out {count} times");
719 tracing::warn!(
720 task_id,
721 count,
722 max,
723 "timeout retries exhausted, blocking task"
724 );
725 if let Err(e) = store::store_set_result(
726 store,
727 repo,
728 task_id,
729 &[
730 (
731 "last_error",
732 serde_json::json!(format!("{error_message} ({block_reason})")),
733 ),
734 ("block_reason", serde_json::json!(block_reason)),
735 ("limit_reroute_chain", serde_json::json!("")),
736 ],
737 )
738 .await
739 {
740 tracing::warn!(task_id, err = %e, "failed to write timeout block fields to store");
741 }
742 let details = serde_json::json!({
743 "failure_reason": "timeout",
744 "error_message": error_message,
745 "timeout_retries": count,
746 });
747 store::store_log_activity(
748 store,
749 repo,
750 task_id,
751 "rerouted",
752 Some("in_progress"),
753 Some("blocked"),
754 None::<&str>,
755 None::<&str>,
756 Some(&details),
757 )
758 .await;
759 Some("blocked".to_string())
760}
761
762pub async fn get_reroute_chain(
764 task_id: &str,
765 store: &Option<Arc<TaskStore>>,
766 repo: &str,
767) -> String {
768 store::opt_store_get_task(store, repo, task_id)
769 .await
770 .map(|t| t.limit_reroute_chain)
771 .unwrap_or_default()
772 .trim()
773 .to_string()
774}
775
776pub async fn update_reroute_chain(
778 task_id: &str,
779 current_agent: &str,
780 existing_chain: &str,
781 store: &Option<Arc<TaskStore>>,
782 repo: &str,
783) -> String {
784 let mut chain = existing_chain.to_string();
785 if chain.is_empty() {
786 chain = current_agent.to_string();
787 } else if !chain.split(',').any(|a| a == current_agent) {
788 chain = format!("{chain},{current_agent}");
789 }
790
791 if let Err(e) = store::store_set_result(
792 store,
793 repo,
794 task_id,
795 &[("limit_reroute_chain", serde_json::json!(chain))],
796 )
797 .await
798 {
799 tracing::warn!(task_id, err = %e, "failed to write limit_reroute_chain to store");
800 }
801 chain
802}
803
804#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
806pub struct ReviewResponse {
807 pub decision: String,
809 pub notes: String,
811 pub test_results: Option<String>,
813 pub issues: Vec<ReviewIssue>,
815}
816
817#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
819pub struct ReviewIssue {
820 pub file: String,
822 pub line: Option<u32>,
824 pub severity: String,
826 pub description: String,
828}
829
830pub fn parse_review_response(text: &str) -> anyhow::Result<ReviewResponse> {
836 let resp = parse_review_json(text)?;
837 if matches!(resp.decision.as_str(), "approve" | "request_changes") {
838 return Ok(resp);
839 }
840
841 let stripped = text.replace("approve|request_changes", "");
844 infer_review_response_from_text(&stripped)
845 .ok_or_else(|| anyhow::anyhow!("invalid review decision: {}", resp.decision))
846}
847
848fn parse_review_json(text: &str) -> anyhow::Result<ReviewResponse> {
849 if let Ok(resp) = serde_json::from_str::<ReviewResponse>(text) {
851 return Ok(resp);
852 }
853
854 if let Some(json_str) = extract_json_block(text)
856 && let Ok(resp) = serde_json::from_str::<ReviewResponse>(&json_str)
857 {
858 return Ok(resp);
859 }
860
861 if let Some(resp) = extract_review_response_object(text) {
862 return Ok(resp);
863 }
864
865 anyhow::bail!("failed to parse review response")
866}
867
868fn infer_review_response_from_text(text: &str) -> Option<ReviewResponse> {
869 let lower = text.to_ascii_lowercase();
870 let has_changes_requested =
871 lower.contains("changes requested") || lower.contains("request_changes");
872 if has_changes_requested {
873 return Some(ReviewResponse {
874 decision: "request_changes".to_string(),
875 notes: "Inferred decision from plain-text review output.".to_string(),
876 test_results: None,
877 issues: Vec::new(),
878 });
879 }
880
881 let positive_approval = lower
884 .split(|c: char| !c.is_ascii_alphabetic())
885 .any(|word| word == "lgtm")
886 || lower.contains("decision is `approve`")
887 || lower.contains("decision: `approve`")
888 || lower.contains("decision is approve")
889 || lower.contains("decision: approve");
890 if positive_approval {
891 return Some(ReviewResponse {
892 decision: "approve".to_string(),
893 notes: "Inferred decision from plain-text review output.".to_string(),
894 test_results: None,
895 issues: Vec::new(),
896 });
897 }
898
899 None
900}
901
902pub fn infer_review_response(text: &str) -> Option<ReviewResponse> {
906 infer_review_response_from_text(text)
907}
908
909fn extract_json_block(text: &str) -> Option<String> {
914 let mut search_from = 0;
915 while let Some(start) = text[search_from..].find("```json") {
916 let abs_start = search_from + start;
917 let after_tag = abs_start + "```json".len();
918 let Some(rest) = text[after_tag..].strip_prefix('\n') else {
919 search_from = after_tag;
920 continue;
921 };
922 let content_start = text.len() - rest.len();
923 let Some(end_offset) = rest.find("```") else {
924 search_from = content_start;
925 continue;
926 };
927 let end = content_start + end_offset;
928 let content = text[content_start..end].trim();
929 if content.starts_with('{') {
930 return Some(content.to_string());
931 }
932 search_from = end + 3;
933 }
934 None
935}
936
937fn extract_review_response_object(text: &str) -> Option<ReviewResponse> {
941 let mut start: Option<usize> = None;
942 let mut depth: i32 = 0;
943 let mut in_string = false;
944 let mut escape = false;
945
946 for (idx, ch) in text.char_indices() {
947 if start.is_some() {
948 if in_string {
949 if escape {
950 escape = false;
951 } else if ch == '\\' {
952 escape = true;
953 } else if ch == '"' {
954 in_string = false;
955 }
956 continue;
957 }
958
959 match ch {
960 '"' => in_string = true,
961 '{' => depth += 1,
962 '}' => {
963 depth -= 1;
964 if depth == 0 {
965 debug_assert!(
966 start.is_some(),
967 "start must be Some when depth returns to 0"
968 );
969 let start_idx = start.unwrap_or(0);
970 let candidate = &text[start_idx..=idx];
971 if let Ok(resp) = serde_json::from_str::<ReviewResponse>(candidate) {
972 return Some(resp);
973 }
974 start = None;
975 }
976 }
977 _ => {}
978 }
979 continue;
980 }
981
982 if ch == '{' {
983 start = Some(idx);
984 depth = 1;
985 in_string = false;
986 escape = false;
987 }
988 }
989
990 None
991}
992
993#[expect(
995 clippy::too_many_arguments,
996 reason = "a memory entry records the task, attempt, agent, model and outcome"
997)]
998pub async fn store_learnings_from_response(
999 task_id: &str,
1000 attempt: u32,
1001 agent: &str,
1002 model: Option<&str>,
1003 response: &AgentResponse,
1004 error: Option<&str>,
1005 store: &Option<Arc<TaskStore>>,
1006 repo: &str,
1007 raw_stdout: &str,
1008) {
1009 let files_modified = if response.files.is_empty() {
1011 let extracted = ndjson::extract_files_changed(raw_stdout);
1012 if !extracted.is_empty() {
1013 tracing::debug!(
1014 task_id,
1015 count = extracted.len(),
1016 "files_modified: fell back to NDJSON extraction (agent reported none)"
1017 );
1018 }
1019 extracted
1020 } else {
1021 response.files.clone()
1022 };
1023
1024 let entry = MemoryEntry {
1026 attempt,
1027 agent: agent.to_string(),
1028 model: model.map(String::from),
1029 learnings: response.learnings.clone(),
1030 error: error.map(String::from),
1031 files_modified,
1032 approach: response.summary.clone(),
1033 timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1034 };
1035
1036 if let Some(st) = store
1037 && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1038 {
1039 match st.append_memory(store_id, &entry).await {
1040 Err(e) => {
1041 tracing::warn!(task_id, error = ?e, "failed to store memory");
1042 }
1043 _ => {
1044 tracing::debug!(task_id, attempt, "stored memory for attempt");
1045 }
1046 }
1047 }
1048}
1049
1050pub async fn store_failure_memory(
1052 task_id: &str,
1053 attempt: u32,
1054 agent: &str,
1055 model: Option<&str>,
1056 error: &str,
1057 store: &Option<Arc<TaskStore>>,
1058 repo: &str,
1059) {
1060 let entry = MemoryEntry {
1061 attempt,
1062 agent: agent.to_string(),
1063 model: model.map(String::from),
1064 learnings: vec![],
1065 error: Some(error.to_string()),
1066 files_modified: vec![],
1067 approach: String::new(),
1068 timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1069 };
1070
1071 if let Some(st) = store
1072 && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1073 {
1074 match st.append_memory(store_id, &entry).await {
1075 Err(e) => {
1076 tracing::warn!(task_id, error = ?e, "failed to store failure memory");
1077 }
1078 _ => {
1079 tracing::debug!(task_id, attempt, "stored failure memory");
1080 }
1081 }
1082 }
1083}
1084
1085#[cfg(test)]
1086mod tests {
1087 use std::env;
1088 #[cfg(unix)]
1089 use std::os::unix::fs::PermissionsExt;
1090 use std::process;
1091 use std::sync::Arc;
1092
1093 use super::super::TaskRunner;
1094 use super::*;
1095 use crate::backends::{ExternalId, ExternalTask};
1096 use crate::engine::cooldown;
1097 use crate::engine::router::{Router, RouterConfig};
1098 use crate::engine::runner::agents::{AgentError, find_agent_result, patterns};
1099 use crate::store::TaskStatus;
1100 use serial_test::serial;
1101
1102 #[serial(cooldown_state)]
1103 #[test]
1104 fn calculate_backoff_delay_jitter_is_centered() {
1105 let base = config::retry_base_delay_ms();
1109 let max = config::retry_max_delay_ms();
1110
1111 let capped = base.min(max);
1113 #[expect(
1114 clippy::cast_precision_loss,
1115 clippy::cast_possible_truncation,
1116 clippy::cast_sign_loss,
1117 reason = "mirrors the jitter range in calculate_backoff_delay"
1118 )]
1119 let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
1120 let lo = capped.saturating_sub(jitter_range);
1121 let hi = capped + jitter_range;
1122
1123 for _ in 0..200 {
1124 let delay = calculate_backoff_delay(0);
1125 assert!(
1126 delay >= lo && delay <= hi,
1127 "delay {delay} out of expected range [{lo}, {hi}]"
1128 );
1129 }
1130 }
1131
1132 #[serial(cooldown_state)]
1133 #[test]
1134 fn fallback_retry_count_uses_larger_network_retry_streak() {
1135 let task = Task {
1136 id: 1,
1137 external_id: None,
1138 repo: "owner/repo".to_string(),
1139 origin: "internal".to_string(),
1140 title: "test".to_string(),
1141 body: String::new(),
1142 status: TaskStatus::New,
1143 source: String::new(),
1144 source_id: String::new(),
1145 author: String::new(),
1146 url: String::new(),
1147 labels: vec![],
1148 agent: None,
1149 model: None,
1150 complexity: "simple".to_string(),
1151 estimate: 0,
1152 route_reason: String::new(),
1153 agent_profile: String::new(),
1154 route_attempts: 0,
1155 attempts: 0,
1156 branch: String::new(),
1157 worktree: String::new(),
1158 worktree_cleaned: false,
1159 owner_node: None,
1160 summary: String::new(),
1161 last_error: String::new(),
1162 parent_id: None,
1163 block_reason: None,
1164 pr_number: None,
1165 pr_review_context: String::new(),
1166 last_review_ts: String::new(),
1167 review_ts_map: "{}".to_string(),
1168 last_comment_review_ts: String::new(),
1169 merge_conflict_retries: 0,
1170 ci_merge_failures: 0,
1171 pr_create_failures: 0,
1172 push_failures: 0,
1173 network_retries: 3,
1174 timeout_retries: 0,
1175 review_agent_failures: 0,
1176 review_resets: 0,
1177 review_cycles: 0,
1178 review_invocations: 0,
1179 needs_review_refires: 0,
1180 input_tokens: 0,
1181 output_tokens: 0,
1182 total_cost_usd: 0.0,
1183 model_reroute_chain: String::new(),
1184 limit_reroute_chain: "claude".to_string(),
1185 memory: vec![],
1186 delegations: vec![],
1187 auto_unblock_count: 0,
1188 auto_unblock_last_at: String::new(),
1189 auto_unblock_last_reason: String::new(),
1190 ci_recovery_count: 0,
1191 no_code_reroutes: 0,
1192 no_code_last_agent: String::new(),
1193 created_at: String::new(),
1194 updated_at: String::new(),
1195 };
1196
1197 assert_eq!(fallback_retry_count(Some(&task)), 3);
1198 }
1199
1200 #[serial(cooldown_state)]
1201 #[test]
1202 fn retryable_type_str_returns_correct_labels() {
1203 assert_eq!(RetryableError::Timeout.type_str(), "timeout");
1204 assert_eq!(RetryableError::Interrupted.type_str(), "interrupted");
1205 assert_eq!(RetryableError::UsageLimit.type_str(), "rate_limit");
1206 assert_eq!(RetryableError::AuthError.type_str(), "auth_error");
1207 assert_eq!(RetryableError::Failed.type_str(), "failed");
1208 assert_eq!(RetryableError::MissingTooling.type_str(), "failed");
1209 }
1210
1211 #[serial(cooldown_state)]
1212 #[test]
1213 fn is_usage_limit_detects_rate_limit() {
1214 assert!(patterns::detect_rate_limit("Error: rate limit exceeded").is_some());
1215 assert!(patterns::detect_rate_limit("HTTP 429 Too Many Requests").is_some());
1216 assert!(patterns::detect_rate_limit("quota exceeded for model").is_some());
1217 assert!(patterns::detect_context_overflow("context_length_exceeded").is_some());
1219 }
1220
1221 #[serial(cooldown_state)]
1222 #[test]
1223 fn is_usage_limit_rejects_normal_text() {
1224 assert!(patterns::detect_rate_limit("task completed successfully").is_none());
1225 assert!(patterns::detect_rate_limit("").is_none());
1226 }
1227
1228 #[serial(cooldown_state)]
1229 #[test]
1230 fn is_auth_error_detects_common_patterns() {
1231 assert!(patterns::detect_auth_error("401 Unauthorized").is_some());
1232 assert!(patterns::detect_auth_error("invalid api key provided").is_some());
1233 assert!(patterns::detect_auth_error("Your billing plan has expired").is_some());
1234 assert!(patterns::detect_auth_error("Error: 403 Forbidden").is_some());
1235 }
1236
1237 #[serial(cooldown_state)]
1238 #[test]
1239 fn is_auth_error_rejects_normal_text() {
1240 assert!(patterns::detect_auth_error("task completed successfully").is_none());
1241 assert!(patterns::detect_auth_error("").is_none());
1242 }
1243
1244 #[serial(cooldown_state)]
1245 #[test]
1246 fn detect_missing_tooling_finds_known_tools() {
1247 let result = patterns::detect_missing_tool("bun: command not found");
1248 assert!(result.is_some());
1249 if let AgentError::MissingTool { tool } = result.unwrap() {
1250 assert_eq!(tool, "bun");
1251 } else {
1252 panic!("expected MissingTool error");
1253 }
1254
1255 let result = patterns::detect_missing_tool("env: anchor: no such file");
1256 assert!(result.is_some());
1257 if let AgentError::MissingTool { tool } = result.unwrap() {
1258 assert_eq!(tool, "anchor");
1259 } else {
1260 panic!("expected MissingTool error");
1261 }
1262
1263 let result = patterns::detect_missing_tool("spawn docker enoent");
1264 assert!(result.is_some());
1265 if let AgentError::MissingTool { tool } = result.unwrap() {
1266 assert_eq!(tool, "docker");
1267 } else {
1268 panic!("expected MissingTool error");
1269 }
1270 }
1271
1272 #[serial(cooldown_state)]
1273 #[test]
1274 fn detect_missing_tooling_returns_none_for_normal() {
1275 assert!(patterns::detect_missing_tool("everything works fine").is_none());
1276 assert!(patterns::detect_missing_tool("").is_none());
1277 }
1278
1279 #[serial(cooldown_state)]
1282 #[tokio::test]
1283 async fn record_and_check_agent_cooldown() {
1284 cooldown::reset_global_state().await;
1285 let agent = "test_cooldown_agent_1";
1287 assert!(!is_agent_in_cooldown(agent));
1288 record_agent_failure_with_message(agent, "").await;
1289 assert!(is_agent_in_cooldown(agent));
1290 }
1291
1292 #[serial(cooldown_state)]
1293 #[tokio::test]
1294 async fn record_and_check_model_cooldown() {
1295 cooldown::reset_global_state().await;
1296 let agent = "test_cooldown_agent_2";
1297 let model = "test_model_x";
1298 assert!(!is_model_in_cooldown(agent, model));
1299 record_model_failure(agent, model).await;
1300 assert!(is_model_in_cooldown(agent, model));
1301 assert!(!is_model_in_cooldown(agent, "other_model"));
1303 }
1304
1305 #[serial(cooldown_state)]
1306 #[tokio::test]
1307 async fn clear_expired_does_not_remove_fresh_entries() {
1308 cooldown::reset_global_state().await;
1309 let agent = "test_cooldown_agent_3";
1310 record_agent_failure_with_message(agent, "").await;
1311 clear_expired_cooldowns();
1312 assert!(is_agent_in_cooldown(agent));
1314 }
1315
1316 #[serial(cooldown_state)]
1318 #[test]
1319 fn pick_fallback_skips_current_agent() {
1320 let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1321 let result = pick_fallback_agent("test_agent_a", "", &available);
1322 assert_eq!(result, Some("test_agent_b".to_string()));
1323 }
1324
1325 #[serial(cooldown_state)]
1326 #[test]
1327 fn pick_fallback_skips_chain_agents() {
1328 let available = vec![
1329 "test_agent_a".to_string(),
1330 "test_agent_b".to_string(),
1331 "test_agent_c".to_string(),
1332 ];
1333 let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1334 assert_eq!(result, Some("test_agent_c".to_string()));
1335 }
1336
1337 #[serial(cooldown_state)]
1338 #[test]
1339 fn pick_fallback_returns_none_when_exhausted() {
1340 let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1341 let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1342 assert!(result.is_none());
1343 }
1344
1345 #[serial(cooldown_state)]
1346 #[test]
1347 fn weight_signal_variants() {
1348 let success = WeightSignal::Success {
1349 agent: "claude".to_string(),
1350 };
1351 let limited = WeightSignal::RateLimited {
1352 agent: "codex".to_string(),
1353 };
1354 let none = WeightSignal::None;
1355
1356 assert!(format!("{success:?}").contains("claude"));
1358 assert!(format!("{limited:?}").contains("codex"));
1359 assert!(format!("{none:?}").contains("None"));
1360 }
1361
1362 #[serial(cooldown_state)]
1363 #[test]
1364 fn parse_review_response_direct_json() {
1365 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1366 let resp = parse_review_response(json).unwrap();
1367 assert_eq!(resp.decision, "approve");
1368 assert_eq!(resp.notes, "LGTM");
1369 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1370 }
1371
1372 #[serial(cooldown_state)]
1373 #[test]
1374 fn parse_review_response_from_markdown() {
1375 let md = r#"Here is my review:
1376
1377```json
1378{"decision":"request_changes","notes":"Fix the bug","issues":[{"file":"src/main.rs","line":10,"severity":"error","description":"null deref"}]}
1379```
1380
1381That's all."#;
1382 let resp = parse_review_response(md).unwrap();
1383 assert_eq!(resp.decision, "request_changes");
1384 assert_eq!(resp.issues.len(), 1);
1385 assert_eq!(resp.issues[0].file, "src/main.rs");
1386 }
1387
1388 #[serial(cooldown_state)]
1389 #[test]
1390 fn parse_review_response_embedded_json_object() {
1391 let text = "Review complete.\n\n{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}\nThanks!";
1392 let resp = parse_review_response(text).unwrap();
1393 assert_eq!(resp.decision, "approve");
1394 assert_eq!(resp.notes, "LGTM");
1395 }
1396
1397 #[serial(cooldown_state)]
1398 #[test]
1399 fn parse_review_response_template_echo_infers_from_notes() {
1400 let json = r#"{"decision":"approve|request_changes","notes":"LGTM","issues":[]}"#;
1401 assert_eq!(parse_review_response(json).unwrap().decision, "approve");
1402
1403 let no_signal = r#"{"decision":"approve|request_changes","notes":"hmm","issues":[]}"#;
1404 assert!(parse_review_response(no_signal).is_err());
1405 }
1406
1407 #[serial(cooldown_state)]
1408 #[test]
1409 fn parse_review_response_invalid() {
1410 let result = parse_review_response("not json at all");
1411 assert!(result.is_err());
1412 }
1413
1414 #[serial(cooldown_state)]
1415 #[test]
1416 fn extract_json_block_basic() {
1417 let md = "text\n```json\n{\"key\": \"value\"}\n```\nmore";
1418 let result = extract_json_block(md);
1419 assert!(result.is_some());
1420 assert!(result.unwrap().contains("\"key\""));
1421 }
1422
1423 #[serial(cooldown_state)]
1424 #[test]
1425 fn extract_json_block_skips_non_json_blocks() {
1426 let md = "```json\nnot-json-array\n```\n\n```json\n{\"real\": true}\n```";
1427 let result = extract_json_block(md);
1428 assert!(result.is_some());
1429 assert!(result.unwrap().contains("\"real\""));
1430 }
1431
1432 #[serial(cooldown_state)]
1433 #[test]
1434 fn extract_json_block_none_when_missing() {
1435 assert!(extract_json_block("no code blocks here").is_none());
1436 }
1437
1438 #[serial(cooldown_state)]
1439 #[test]
1440 fn extract_json_block_skips_malformed_intermediate_blocks() {
1441 let md = "prefix ```json{\"broken\": true}\n\n```json\n{\"real\": true}\n```";
1442 let result = extract_json_block(md);
1443 assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1444 }
1445
1446 #[serial(cooldown_state)]
1447 #[test]
1448 fn extract_json_block_skips_multiple_malformed_intermediate_blocks() {
1449 let md = "prefix ```json{\"broken\": true}\nmid ```json[1, 2, 3]\n\n```json\n{\"real\": true}\n```";
1450 let result = extract_json_block(md);
1451 assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1452 }
1453
1454 fn parse_review_plain(text: &str) -> anyhow::Result<ReviewResponse> {
1459 parse_review_response(text).or_else(|_| {
1460 infer_review_response(text)
1461 .ok_or_else(|| anyhow::anyhow!("failed to parse review response"))
1462 })
1463 }
1464
1465 #[serial(cooldown_state)]
1466 #[test]
1467 fn parse_review_from_output_direct_json() {
1468 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1469 let resp = parse_review_response(json).unwrap();
1470 assert_eq!(resp.decision, "approve");
1471 }
1472
1473 #[serial(cooldown_state)]
1474 #[test]
1475 fn parse_review_from_output_markdown() {
1476 let md = "Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}\n```\n";
1477 let resp = parse_review_response(md).unwrap();
1478 assert_eq!(resp.decision, "request_changes");
1479 }
1480
1481 #[serial(cooldown_state)]
1482 #[test]
1483 fn parse_review_from_output_plain_text_fallback() {
1484 let text = "The review is done. LGTM.";
1485 let resp = parse_review_plain(text).unwrap();
1486 assert_eq!(resp.decision, "approve");
1487 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1488 }
1489
1490 #[serial(cooldown_state)]
1492 #[test]
1493 fn parse_review_from_output_ndjson_part_format() {
1494 let ndjson = concat!(
1495 r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1496 "\n",
1497 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Reviewing the PR...\n\n```json\n{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1498 "\n",
1499 r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop"}}"#,
1500 );
1501 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1502 assert_eq!(resp.decision, "approve");
1503 assert_eq!(resp.notes, "All checks pass");
1504 }
1505
1506 #[serial(cooldown_state)]
1508 #[test]
1509 fn parse_review_from_output_ndjson_direct_text_format() {
1510 let ndjson = concat!(
1511 r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1512 "\n",
1513 r#"{"type":"text","timestamp":1001,"sessionID":"ses_abc","text":"Here is my review:\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the bug\",\"test_results\":\"fail\",\"issues\":[]}\n```\n"}"#,
1514 "\n",
1515 r#"{"type":"step_finish","timestamp":1002,"sessionID":"ses_abc"}"#,
1516 );
1517 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1518 assert_eq!(resp.decision, "request_changes");
1519 assert_eq!(resp.notes, "Fix the bug");
1520 }
1521
1522 #[serial(cooldown_state)]
1524 #[test]
1525 fn parse_review_from_output_ndjson_direct_json_in_text_event() {
1526 let ndjson = concat!(
1527 r#"{"type":"step_start","timestamp":1000}"#,
1528 "\n",
1529 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1530 "\n",
1531 r#"{"type":"step_finish","timestamp":1002}"#,
1532 );
1533 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1534 assert_eq!(resp.decision, "approve");
1535 }
1536
1537 #[serial(cooldown_state)]
1538 #[test]
1539 fn parse_review_from_output_ndjson_prefers_final_json_text_event() {
1540 let ndjson = concat!(
1541 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Working on it..."}}"#,
1542 "\n",
1543 r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1544 );
1545 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1546 assert_eq!(resp.decision, "approve");
1547 assert_eq!(resp.notes, "LGTM");
1548 }
1549
1550 #[serial(cooldown_state)]
1552 #[test]
1553 fn parse_review_from_output_ndjson_concatenated_text() {
1554 let ndjson = concat!(
1555 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Here is my\n"}}"#,
1556 "\n",
1557 r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"review:\n\n```json\n{\"decision\":\"approve\",\"notes\":\"OK\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1558 );
1559 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1560 assert_eq!(resp.decision, "approve");
1561 }
1562
1563 #[serial(cooldown_state)]
1568 #[test]
1569 fn parse_review_from_output_codex_agent_message() {
1570 let ndjson = concat!(
1571 r#"{"type":"thread.started","thread_id":"t1"}"#,
1572 "\n",
1573 r#"{"type":"turn.started"}"#,
1574 "\n",
1575 r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Planning git commands..."}}"#,
1576 "\n",
1577 r#"{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"git log --oneline -5","output":"abc123 fix: something"}}"#,
1578 "\n",
1579 r#"{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1580 "\n",
1581 r#"{"type":"turn.completed"}"#,
1582 );
1583 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1584 assert_eq!(resp.decision, "approve");
1585 assert_eq!(resp.notes, "All checks pass");
1586 }
1587
1588 #[serial(cooldown_state)]
1590 #[test]
1591 fn parse_review_from_output_codex_agent_message_markdown() {
1592 let ndjson = concat!(
1593 r#"{"type":"thread.started","thread_id":"t1"}"#,
1594 "\n",
1595 r#"{"type":"turn.started"}"#,
1596 "\n",
1597 r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Reviewing..."}}"#,
1598 "\n",
1599 r#"{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the tests\",\"issues\":[]}\n```\n"}}"#,
1600 "\n",
1601 r#"{"type":"turn.completed"}"#,
1602 );
1603 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1604 assert_eq!(resp.decision, "request_changes");
1605 assert_eq!(resp.notes, "Fix the tests");
1606 }
1607
1608 #[serial(cooldown_state)]
1610 #[test]
1611 fn codex_skips_reasoning_items() {
1612 let ndjson = concat!(
1613 r#"{"type":"item.completed","item":{"type":"reasoning","text":"internal thoughts..."}}"#,
1614 "\n",
1615 r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}}"#,
1616 );
1617 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1618 assert_eq!(resp.decision, "approve");
1619 assert_eq!(resp.notes, "LGTM");
1620 }
1621
1622 #[serial(cooldown_state)]
1624 #[test]
1625 fn infer_review_response_negation_not_approved() {
1626 assert!(
1627 parse_review_plain("The changes were not approved.").is_err(),
1628 "\"not approved\" should not infer approval"
1629 );
1630 }
1631
1632 #[serial(cooldown_state)]
1633 #[test]
1634 fn infer_review_response_negation_unapproved() {
1635 assert!(
1636 parse_review_plain("Unapproved changes found in the diff.").is_err(),
1637 "\"unapproved\" should not infer approval"
1638 );
1639 }
1640
1641 #[serial(cooldown_state)]
1642 #[test]
1643 fn infer_review_response_negation_not_approving() {
1644 assert!(
1645 parse_review_plain("I am not approving this PR.").is_err(),
1646 "\"not approving\" should not infer approval"
1647 );
1648 }
1649
1650 #[serial(cooldown_state)]
1651 #[test]
1652 fn infer_review_response_negation_cannot_be_approved() {
1653 assert!(
1654 parse_review_plain("This PR cannot be approved until the race in tick.rs is fixed.")
1655 .is_err(),
1656 "\"cannot be approved\" should not infer approval"
1657 );
1658 }
1659
1660 #[serial(cooldown_state)]
1661 #[test]
1662 fn infer_review_response_negation_should_not_be_approved() {
1663 assert!(
1664 parse_review_plain("The diff breaks the build and should not be approved.").is_err(),
1665 "\"should not be approved\" should not infer approval"
1666 );
1667 }
1668
1669 #[serial(cooldown_state)]
1670 #[test]
1671 fn infer_review_response_negation_not_yet_approved() {
1672 assert!(
1673 parse_review_plain("This change is not yet approved.").is_err(),
1674 "\"not yet approved\" should not infer approval"
1675 );
1676 }
1677
1678 #[serial(cooldown_state)]
1679 #[test]
1680 fn infer_review_response_negation_wont_approve() {
1681 assert!(
1682 parse_review_plain("I won't approve this until tests cover the retry path.").is_err(),
1683 "\"won't approve\" should not infer approval"
1684 );
1685 }
1686
1687 #[serial(cooldown_state)]
1688 #[test]
1689 fn infer_review_response_bare_approved() {
1690 assert!(
1691 parse_review_plain("The PR was approved.").is_err(),
1692 "bare \"approved\" should not infer approval"
1693 );
1694 }
1695
1696 #[serial(cooldown_state)]
1698 #[test]
1699 fn infer_review_response_lgtm() {
1700 let resp = parse_review_plain("LGTM, everything looks fine.").unwrap();
1701 assert_eq!(resp.decision, "approve");
1702 }
1703
1704 #[serial(cooldown_state)]
1706 #[test]
1707 fn infer_review_response_looks_good() {
1708 assert!(
1709 parse_review_plain("Looks good to me!").is_err(),
1710 "\"looks good\" should not infer approval"
1711 );
1712 }
1713
1714 #[serial(cooldown_state)]
1715 #[test]
1716 fn infer_review_response_all_checks_passed() {
1717 assert!(
1718 parse_review_plain(
1719 "The background task completed but I already got the CI results directly. All checks passed — no action needed.",
1720 )
1721 .is_err(),
1722 "\"all checks passed\" should not infer approval"
1723 );
1724 }
1725
1726 #[serial(cooldown_state)]
1727 #[test]
1728 fn infer_review_response_checks_passed() {
1729 assert!(
1730 parse_review_plain("All checks passed (fmt ✓, clippy ✓, tests ✓).").is_err(),
1731 "\"all checks passed\" should not infer approval"
1732 );
1733 }
1734
1735 #[serial(cooldown_state)]
1736 #[test]
1737 fn infer_review_response_all_tests_passed() {
1738 assert!(
1739 parse_review_plain("All tests passed and the implementation looks correct.").is_err(),
1740 "\"all tests passed\" should not infer approval"
1741 );
1742 }
1743
1744 #[serial(cooldown_state)]
1745 #[test]
1746 fn infer_review_response_no_issues_found() {
1747 assert!(
1748 parse_review_plain("Review complete. No issues found in the diff.").is_err(),
1749 "\"no issues found\" should not infer approval"
1750 );
1751 }
1752
1753 #[serial(cooldown_state)]
1755 #[test]
1756 fn infer_review_response_partial_tests_passed_no_approve() {
1757 let text = "The existing tests passed before this change but the new payment logic has a null-pointer risk on line 42.";
1758 assert!(parse_review_plain(text).is_err());
1759 }
1760
1761 #[serial(cooldown_state)]
1763 #[test]
1764 fn infer_review_response_partial_checks_passed_no_approve() {
1765 let text = "CI checks passed for the base branch but this PR adds a broken path.";
1766 assert!(parse_review_plain(text).is_err());
1767 }
1768
1769 #[serial(cooldown_state)]
1771 #[test]
1772 fn infer_review_response_decision_is_approve_backtick() {
1773 let text = "Already retrieved the output and completed the review above. The CI test check passed (3m11s) and all other checks are green. Decision is `approve`.";
1774 let resp = parse_review_plain(text).unwrap();
1775 assert_eq!(resp.decision, "approve");
1776 }
1777
1778 #[serial(cooldown_state)]
1780 #[test]
1781 fn infer_review_response_decision_colon_approve() {
1782 let text = "Reviewed the PR. No blocking issues found. Decision: approve";
1783 let resp = parse_review_plain(text).unwrap();
1784 assert_eq!(resp.decision, "approve");
1785 }
1786
1787 #[serial(cooldown_state)]
1790 #[test]
1791 fn infer_review_response_negation_do_not_approve_backtick() {
1792 let text = "There are unresolved issues — do not `approve` this PR until they are fixed.";
1793 assert!(
1794 parse_review_plain(text).is_err(),
1795 "negative context with backtick approve must not infer approval"
1796 );
1797 }
1798
1799 fn parse_via_agent_path(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
1804 let text = find_agent_result(agent, output)
1805 .map(|r| r.result_text)
1806 .filter(|t| !t.is_empty())
1807 .unwrap_or_else(|| output.to_string());
1808 if let Ok(r) = parse_review_response(&text) {
1809 return Ok(r);
1810 }
1811 infer_review_response_from_text(&text)
1812 .ok_or_else(|| anyhow::anyhow!("failed to parse review response from {agent} output"))
1813 }
1814
1815 #[serial(cooldown_state)]
1816 #[test]
1817 fn agent_output_claude_ndjson_review() {
1818 let ndjson = concat!(
1819 r#"{"type":"system","subtype":"init","session_id":"s1"}"#,
1820 "\n",
1821 r#"{"type":"assistant","message":{"content":[{"type":"text","text":"Reviewing..."}]}}"#,
1822 "\n",
1823 r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}","usage":{"input_tokens":100,"output_tokens":20}}"#,
1824 );
1825 let resp = parse_via_agent_path("claude", ndjson).unwrap();
1826 assert_eq!(resp.decision, "approve");
1827 }
1828
1829 #[serial(cooldown_state)]
1830 #[test]
1831 fn agent_output_opencode_ndjson_review() {
1832 let ndjson = concat!(
1833 r#"{"type":"step_start","timestamp":1000}"#,
1834 "\n",
1835 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}"}}"#,
1836 "\n",
1837 r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop","tokens":{"input":100,"output":10}}}"#,
1838 );
1839 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1840 assert_eq!(resp.decision, "request_changes");
1841 }
1842
1843 #[serial(cooldown_state)]
1844 #[test]
1845 fn agent_output_codex_ndjson_review() {
1846 let ndjson = concat!(
1847 r#"{"type":"thread.started","thread_id":"t1"}"#,
1848 "\n",
1849 r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All good\",\"issues\":[]}"}}"#,
1850 "\n",
1851 r#"{"type":"turn.completed"}"#,
1852 );
1853 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1854 assert_eq!(resp.decision, "approve");
1855 }
1856
1857 #[serial(cooldown_state)]
1858 #[test]
1859 fn agent_output_plain_text_keyword_fallback() {
1860 let text = "All checks passed, LGTM.";
1861 let resp = parse_via_agent_path("claude", text).unwrap();
1862 assert_eq!(resp.decision, "approve");
1863 }
1864
1865 #[serial(cooldown_state)]
1866 #[test]
1867 fn agent_output_direct_json() {
1868 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1869 let resp = parse_via_agent_path("opencode", json).unwrap();
1870 assert_eq!(resp.decision, "approve");
1871 }
1872
1873 #[serial(cooldown_state)]
1877 #[test]
1878 fn agent_output_text_with_json_fragment_no_false_positive() {
1879 let text = r#"I checked the file and found {"type":"text"} in the output. No issues."#;
1880 let result = parse_via_agent_path("claude", text);
1884 assert!(
1886 result.is_err() || result.as_ref().is_ok_and(|r| r.decision != "text"),
1887 "should not extract review from embedded JSON fragment"
1888 );
1889 }
1890
1891 #[serial(cooldown_state)]
1894 #[test]
1895 fn parse_review_response_ignores_prose_with_decision_word() {
1896 let text = r#"The decision was made. Here is a debug trace: {"trace":123}"#;
1897 assert!(
1898 parse_review_response(text).is_err(),
1899 "prose with 'decision' word and unrelated JSON should not parse as ReviewResponse"
1900 );
1901 }
1902
1903 #[serial(cooldown_state)]
1906 #[test]
1907 fn parse_review_response_ignores_embedded_json_without_decision_key() {
1908 let text = r#"Analysis complete. Config: {"timeout":30,"retries":3}. Done."#;
1909 assert!(
1910 parse_review_response(text).is_err(),
1911 "embedded JSON without 'decision' key should not parse as ReviewResponse"
1912 );
1913 }
1914
1915 #[serial(cooldown_state)]
1918 #[test]
1919 fn read_file_with_limit_reads_small_file_completely() {
1920 let temp_dir = env::temp_dir();
1921 let file_path = temp_dir.join(format!("orch_test_small_{}.txt", process::id()));
1922 let content = "Hello, World!";
1923 fs::write(&file_path, content).unwrap();
1924
1925 let result = read_file_with_limit(&file_path, 1024);
1926 assert!(result.is_ok());
1927 assert_eq!(result.unwrap(), content);
1928
1929 let _ = fs::remove_file(&file_path);
1930 }
1931
1932 #[serial(cooldown_state)]
1933 #[test]
1934 fn read_file_with_limit_truncates_large_file() {
1935 let temp_dir = env::temp_dir();
1936 let file_path = temp_dir.join(format!("orch_test_large_{}.txt", process::id()));
1937
1938 let large_content = "x".repeat(200);
1940 fs::write(&file_path, &large_content).unwrap();
1941
1942 let result = read_file_with_limit(&file_path, 50);
1944 assert!(result.is_ok());
1945
1946 let read_content = result.unwrap();
1947 assert_eq!(read_content.len(), 50);
1949 assert_eq!(read_content, "x".repeat(50));
1950
1951 let _ = fs::remove_file(&file_path);
1952 }
1953
1954 #[serial(cooldown_state)]
1955 #[test]
1956 fn read_file_with_limit_handles_empty_file() {
1957 let temp_dir = env::temp_dir();
1958 let file_path = temp_dir.join(format!("orch_test_empty_{}.txt", process::id()));
1959 fs::write(&file_path, "").unwrap();
1960
1961 let result = read_file_with_limit(&file_path, 1024);
1962 assert!(result.is_ok());
1963 assert_eq!(result.unwrap(), "");
1964
1965 let _ = fs::remove_file(&file_path);
1966 }
1967
1968 #[serial(cooldown_state)]
1969 #[test]
1970 fn read_file_with_limit_handles_nonexistent_file() {
1971 let temp_dir = env::temp_dir();
1972 let file_path = temp_dir.join("orch_test_nonexistent_xyz123.txt");
1973
1974 let result = read_file_with_limit(&file_path, 1024);
1975 assert!(result.is_err());
1976 }
1977
1978 #[serial(cooldown_state)]
1979 #[test]
1980 fn read_file_with_limit_reads_exact_limit() {
1981 let temp_dir = env::temp_dir();
1982 let file_path = temp_dir.join(format!("orch_test_exact_{}.txt", process::id()));
1983
1984 let content = "y".repeat(100);
1986 fs::write(&file_path, &content).unwrap();
1987
1988 let result = read_file_with_limit(&file_path, 100);
1990 assert!(result.is_ok());
1991 assert_eq!(result.unwrap().len(), 100);
1992
1993 let _ = fs::remove_file(&file_path);
1994 }
1995
1996 #[serial(cooldown_state)]
1997 #[test]
1998 fn read_file_with_limit_handles_unicode_truncation() {
1999 let temp_dir = env::temp_dir();
2000 let file_path = temp_dir.join(format!("orch_test_unicode_{}.txt", process::id()));
2001
2002 let emoji = "🎉".repeat(100); fs::write(&file_path, &emoji).unwrap();
2005
2006 let result = read_file_with_limit(&file_path, 50);
2008 assert!(result.is_ok());
2009
2010 let read_content = result.unwrap();
2011 assert_ne!(read_content, "");
2013
2014 let _ = fs::remove_file(&file_path);
2015 }
2016
2017 fn parse_review_via_agent(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
2021 let text = find_agent_result(agent, output)
2022 .map(|r| r.result_text)
2023 .filter(|t| !t.is_empty())
2024 .unwrap_or_else(|| output.to_string());
2025 if let Ok(r) = parse_review_response(&text) {
2026 return Ok(r);
2027 }
2028 infer_review_response(&text)
2029 .ok_or_else(|| anyhow::anyhow!("failed to parse review from {agent} output"))
2030 }
2031
2032 #[serial(cooldown_state)]
2035 #[test]
2036 fn fixture_claude_review_approve() {
2037 let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2038 let resp = parse_review_via_agent("claude", raw).unwrap();
2039 assert_eq!(resp.decision, "approve");
2040 assert_eq!(resp.notes, "All checks pass");
2041 assert_eq!(resp.test_results.as_deref(), Some("pass"));
2042 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2043 }
2044
2045 #[serial(cooldown_state)]
2046 #[test]
2047 fn fixture_claude_review_request_changes() {
2048 let raw = include_str!("../../../tests/fixtures/review_claude_request_changes.jsonl");
2049 let resp = parse_review_via_agent("claude", raw).unwrap();
2050 assert_eq!(resp.decision, "request_changes");
2051 assert_eq!(resp.notes, "Fix the null deref on line 42");
2052 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2053 assert_eq!(resp.issues.len(), 1);
2054 assert_eq!(resp.issues[0].file, "src/main.rs");
2055 assert_eq!(resp.issues[0].line, Some(42));
2056 assert_eq!(resp.issues[0].severity, "error");
2057 }
2058
2059 #[serial(cooldown_state)]
2060 #[test]
2061 fn fixture_claude_review_rate_limit_returns_error() {
2062 let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2065 let result = find_agent_result("claude", raw);
2066 assert!(
2067 result.is_some(),
2068 "should find result even for error envelope"
2069 );
2070 let result = result.unwrap();
2071 assert!(result.is_error, "rate limit should be flagged as error");
2072 assert!(result.result_text.contains("rate limit"));
2073 }
2074
2075 #[serial(cooldown_state)]
2078 #[test]
2079 fn fixture_opencode_review_approve() {
2080 let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2081 let resp = parse_review_via_agent("opencode", raw).unwrap();
2082 assert_eq!(resp.decision, "approve");
2083 assert_eq!(resp.notes, "All checks pass");
2084 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2085 }
2086
2087 #[serial(cooldown_state)]
2088 #[test]
2089 fn fixture_opencode_review_request_changes() {
2090 let raw = include_str!("../../../tests/fixtures/review_opencode_request_changes.jsonl");
2091 let resp = parse_review_via_agent("opencode", raw).unwrap();
2092 assert_eq!(resp.decision, "request_changes");
2093 assert_eq!(resp.notes, "Fix the memory leak");
2094 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2095 assert_eq!(resp.issues.len(), 1);
2096 assert_eq!(resp.issues[0].file, "src/lib.rs");
2097 assert_eq!(resp.issues[0].line, Some(100));
2098 }
2099
2100 #[serial(cooldown_state)]
2101 #[test]
2102 fn fixture_opencode_review_plain_text_approve() {
2103 let raw = include_str!("../../../tests/fixtures/review_opencode_plain_text.jsonl");
2105 let resp = parse_review_via_agent("opencode", raw).unwrap();
2106 assert_eq!(resp.decision, "approve");
2107 }
2108
2109 #[serial(cooldown_state)]
2112 #[test]
2113 fn fixture_codex_review_approve() {
2114 let raw = include_str!("../../../tests/fixtures/review_codex_approve.jsonl");
2115 let resp = parse_review_via_agent("codex", raw).unwrap();
2116 assert_eq!(resp.decision, "approve");
2117 assert_eq!(resp.notes, "Clean implementation");
2118 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2119 }
2120
2121 #[serial(cooldown_state)]
2122 #[test]
2123 fn fixture_codex_review_request_changes() {
2124 let raw = include_str!("../../../tests/fixtures/review_codex_request_changes.jsonl");
2125 let resp = parse_review_via_agent("codex", raw).unwrap();
2126 assert_eq!(resp.decision, "request_changes");
2127 assert_eq!(resp.notes, "Race condition on line 15");
2128 assert_eq!(resp.issues.len(), 1);
2129 assert_eq!(resp.issues[0].file, "src/server.rs");
2130 assert_eq!(resp.issues[0].line, Some(15));
2131 }
2132
2133 #[serial(cooldown_state)]
2134 #[test]
2135 fn fixture_codex_review_plain_text_approve() {
2136 let raw = include_str!("../../../tests/fixtures/review_codex_plain_text.jsonl");
2138 let resp = parse_review_via_agent("codex", raw).unwrap();
2139 assert_eq!(resp.decision, "approve");
2140 }
2141
2142 #[serial(cooldown_state)]
2145 #[test]
2146 fn fixture_kimi_review_approve() {
2147 let raw = include_str!("../../../tests/fixtures/review_kimi_approve.jsonl");
2148 let resp = parse_review_via_agent("kimi", raw).unwrap();
2149 assert_eq!(resp.decision, "approve");
2150 assert_eq!(resp.notes, "LGTM");
2151 assert_eq!(resp.test_results.as_deref(), Some("pass"));
2152 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2153 }
2154
2155 #[serial(cooldown_state)]
2158 #[test]
2159 fn fixture_minimax_review_request_changes() {
2160 let raw = include_str!("../../../tests/fixtures/review_minimax_request_changes.jsonl");
2161 let resp = parse_review_via_agent("minimax", raw).unwrap();
2162 assert_eq!(resp.decision, "request_changes");
2163 assert_eq!(resp.notes, "Fix the bug");
2164 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2165 assert_eq!(resp.issues.len(), 1);
2166 assert_eq!(resp.issues[0].file, "src/foo.rs");
2167 assert_eq!(resp.issues[0].line, Some(42));
2168 assert_eq!(resp.issues[0].severity, "error");
2169 }
2170
2171 #[serial(cooldown_state)]
2175 #[test]
2176 fn find_agent_result_extracts_tokens_claude() {
2177 let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2178 let result = find_agent_result("claude", raw).expect("should find result");
2179 assert_eq!(result.input_tokens, Some(200));
2180 assert_eq!(result.output_tokens, Some(30));
2181 assert!(result.cost_usd.is_none()); }
2183
2184 #[serial(cooldown_state)]
2185 #[test]
2186 fn find_agent_result_extracts_tokens_opencode() {
2187 let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2188 let result = find_agent_result("opencode", raw).expect("should find result");
2189 assert_eq!(result.input_tokens, Some(2900));
2190 assert_eq!(result.output_tokens, Some(100));
2191 assert_eq!(result.cost_usd, Some(0.002));
2192 }
2193
2194 #[serial(cooldown_state)]
2196 #[test]
2197 fn find_agent_result_returns_none_for_empty() {
2198 assert!(find_agent_result("claude", "").is_none());
2199 assert!(find_agent_result("opencode", "").is_none());
2200 assert!(find_agent_result("codex", "").is_none());
2201 assert!(find_agent_result("kimi", "").is_none());
2202 assert!(find_agent_result("minimax", "").is_none());
2203 }
2204
2205 #[serial(cooldown_state)]
2206 #[test]
2207 fn find_agent_result_returns_none_for_plain_text() {
2208 assert!(find_agent_result("claude", "just some plain text").is_none());
2209 assert!(find_agent_result("codex", "just some plain text").is_none());
2210 }
2213
2214 #[serial(cooldown_state)]
2218 #[test]
2219 fn find_agent_result_propagates_claude_rate_limit() {
2220 let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2221 let result = find_agent_result("claude", raw).expect("should find result");
2222 assert!(result.is_error, "should be flagged as error");
2223 assert!(result.result_text.contains("rate limit"));
2224 }
2225
2226 #[cfg(unix)]
2229 fn failover_env() -> tempfile::TempDir {
2230 let dir = tempfile::tempdir().unwrap();
2231 let bin = dir.path().join("bin");
2232 fs::create_dir(&bin).unwrap();
2233 for agent in ["claude", "codex"] {
2234 let path = bin.join(agent);
2235 fs::write(&path, "#!/bin/sh\n").unwrap();
2236 fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap();
2237 }
2238 let home = dir.path().join("home");
2239 fs::create_dir(&home).unwrap();
2240 fs::write(
2241 home.join("config.yml"),
2242 "agents: [claude, codex]\ndispatch:\n retry_base_delay_ms: 1\n retry_max_delay_ms: 1\nworkflow:\n max_timeout_retries: 3\n",
2243 )
2244 .unwrap();
2245 let mut path = bin.as_os_str().to_os_string();
2246 path.push(":");
2247 path.push(env::var_os("PATH").unwrap_or_default());
2248 unsafe { env::set_var("PATH", path) };
2250 unsafe { env::set_var("ORCH_HOME", &home) };
2252 cmd_cache::reset_for_test();
2253 dir
2254 }
2255
2256 async fn failover_task() -> (Arc<TaskStore>, String) {
2257 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
2258 let id = store
2259 .create_internal("owner/repo", "t", "b", "test", "", None)
2260 .await
2261 .unwrap();
2262 (store, format!("internal:{id}"))
2263 }
2264
2265 #[cfg(unix)]
2267 #[serial(cooldown_state)]
2268 #[tokio::test]
2269 async fn timeout_with_every_other_agent_cooled_retries_as_new() {
2270 let _env = failover_env();
2271 cooldown::reset_global_state().await;
2272 set_agent_cooldown("codex", 600).await;
2273 let (store, task_id) = failover_task().await;
2274 let store = Some(store);
2275
2276 let outcome = handle_failover(
2277 &task_id,
2278 "claude",
2279 RetryableError::Timeout,
2280 "claude timed out after 1804s",
2281 false,
2282 &store,
2283 "owner/repo",
2284 )
2285 .await;
2286
2287 assert_eq!(outcome.status, "new");
2288 assert!(outcome.next_agent.is_none());
2289 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2290 .await
2291 .unwrap();
2292 assert!(task.limit_reroute_chain.is_empty(), "chain must be cleared");
2293 assert!(
2294 task.last_error.contains("will retry"),
2295 "{}",
2296 task.last_error
2297 );
2298 assert_eq!(task.timeout_retries, 1);
2299 }
2300
2301 #[cfg(unix)]
2303 #[serial(cooldown_state)]
2304 #[tokio::test]
2305 async fn timeout_with_a_free_agent_reroutes_to_it() {
2306 let _env = failover_env();
2307 cooldown::reset_global_state().await;
2308 let (store, task_id) = failover_task().await;
2309 let store = Some(store);
2310
2311 let outcome = handle_failover(
2312 &task_id,
2313 "claude",
2314 RetryableError::Timeout,
2315 "claude timed out after 1804s",
2316 false,
2317 &store,
2318 "owner/repo",
2319 )
2320 .await;
2321
2322 assert_eq!(outcome.status, "new");
2323 assert_eq!(outcome.next_agent.as_deref(), Some("codex"));
2324 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2325 .await
2326 .unwrap();
2327 assert_eq!(task.limit_reroute_chain, "claude");
2328 assert_eq!(task.agent.as_deref(), Some("codex"));
2329 assert!(
2330 task.model.as_deref().unwrap_or("").is_empty(),
2331 "model must be cleared so routing picks one for the fallback, got {:?}",
2332 task.model
2333 );
2334 assert!(
2335 task.last_error.contains("switching to codex"),
2336 "{}",
2337 task.last_error
2338 );
2339 }
2340
2341 #[cfg(unix)]
2343 #[serial(cooldown_state)]
2344 #[tokio::test]
2345 async fn timeout_failover_keeps_fallback_after_reroute_clear() {
2346 let _env = failover_env();
2347 cooldown::reset_global_state().await;
2348 let (store_arc, task_id) = failover_task().await;
2349 let store_id = store_arc
2350 .resolve_task_id("owner/repo", &task_id)
2351 .await
2352 .unwrap()
2353 .unwrap();
2354 store_arc
2355 .update_status(store_id, TaskStatus::InProgress)
2356 .await
2357 .unwrap();
2358 store_arc
2359 .set_fields(store_id, &[("agent", serde_json::json!("claude"))])
2360 .await
2361 .unwrap();
2362
2363 let store = Some(Arc::clone(&store_arc));
2364 let outcome = handle_failover(
2365 &task_id,
2366 "claude",
2367 RetryableError::Timeout,
2368 "claude timed out after 1804s",
2369 false,
2370 &store,
2371 "owner/repo",
2372 )
2373 .await;
2374 assert_eq!(outcome.status, "new");
2375 assert_eq!(outcome.next_agent.as_deref(), Some("codex"));
2376
2377 let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store_arc));
2378 runner
2379 .clear_agent_on_reroute_unless_fallback(&task_id, "claude")
2380 .await;
2381
2382 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2383 .await
2384 .unwrap();
2385 assert_eq!(task.agent.as_deref(), Some("codex"));
2386 assert_eq!(task.status, TaskStatus::InProgress);
2387 assert!(
2388 task.model.as_deref().unwrap_or("").is_empty(),
2389 "model must stay empty so routing picks one for the fallback, got {:?}",
2390 task.model
2391 );
2392
2393 let mut config = RouterConfig::default();
2394 let medium = config.model_map.entry("medium".to_string()).or_default();
2395 medium.insert("claude".to_string(), vec!["haiku".to_string()]);
2396 medium.insert("codex".to_string(), vec!["gpt-test".to_string()]);
2397 let agents = vec!["claude".to_string(), "codex".to_string()];
2398 let mut router = Router::new_for_test(config, agents);
2399 let labeled = ExternalTask {
2400 id: ExternalId(task_id.clone()),
2401 title: "Test".to_string(),
2402 body: "Test body".to_string(),
2403 state: "open".to_string(),
2404 labels: vec!["agent:claude".to_string()],
2405 author: "test".to_string(),
2406 created_at: "2026-01-01T00:00:00Z".to_string(),
2407 updated_at: "2026-01-01T00:00:00Z".to_string(),
2408 url: format!("https://github.com/owner/repo/issues/{task_id}"),
2409 };
2410 let result = router
2411 .route(&labeled, &store_arc, "owner/repo")
2412 .await
2413 .unwrap();
2414 assert_eq!(result.agent, "codex");
2415 }
2416
2417 #[cfg(unix)]
2419 #[serial(cooldown_state)]
2420 #[tokio::test]
2421 async fn repeated_timeouts_block_the_task() {
2422 let _env = failover_env();
2423 cooldown::reset_global_state().await;
2424 set_agent_cooldown("codex", 600).await;
2425 let (store, task_id) = failover_task().await;
2426 let store = Some(store);
2427
2428 let mut statuses = Vec::new();
2429 for _ in 0..3 {
2430 statuses.push(
2431 handle_failover(
2432 &task_id,
2433 "claude",
2434 RetryableError::Timeout,
2435 "claude timed out after 1804s",
2436 false,
2437 &store,
2438 "owner/repo",
2439 )
2440 .await
2441 .status,
2442 );
2443 }
2444
2445 assert_eq!(statuses, ["new", "new", "blocked"]);
2446 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2447 .await
2448 .unwrap();
2449 assert_eq!(task.block_reason.as_deref(), Some("timed out 3 times"));
2450 assert_eq!(task.timeout_retries, 3);
2451 }
2452}