1use crate::store;
10use crate::store::TaskStore;
11use std::collections::HashSet;
12use std::collections::hash_map::DefaultHasher;
13use std::fs;
14use std::fs::File;
15use std::hash::Hash;
16use std::hash::Hasher;
17use std::io::Read;
18use std::io::Seek;
19use std::io::SeekFrom;
20use std::path::{Path, PathBuf};
21use std::result;
22use std::sync::Arc;
23use std::thread;
24use std::time::Duration;
25use std::time::SystemTime;
26
27use tokio::fs as tokio_fs;
28use tokio::task;
29use tokio::time;
30use crate::cli::ndjson;
32use crate::cmd_cache;
33use crate::config as app_config;
34use crate::engine;
35pub use crate::engine::cooldown::{
36 clear_expired_cooldowns, is_agent_in_cooldown, is_model_in_cooldown,
37 record_agent_failure_with_message, record_model_failure, set_agent_cooldown,
38};
39use crate::engine::router::config;
40use crate::home;
41use crate::parser::AgentResponse;
42use crate::store::MemoryEntry;
43use crate::store::Task;
44
45const JITTER_FACTOR: f64 = 0.3;
46
47const MAX_OUTPUT_SIZE_BYTES: u64 = 10 * 1024 * 1024;
50
51fn read_file_with_limit<P: AsRef<Path>>(path: P, max_bytes: u64) -> anyhow::Result<String> {
57 let path = path.as_ref();
58 let metadata = fs::metadata(path)?;
59
60 if !metadata.is_file() {
61 anyhow::bail!("not a file: {}", path.display());
62 }
63
64 let file_size = metadata.len();
65
66 if file_size <= max_bytes {
68 return fs::read_to_string(path).map_err(Into::into);
69 }
70
71 let mut file = File::open(path)?;
73 let start_pos = file_size.saturating_sub(max_bytes);
74 file.seek(SeekFrom::Start(start_pos))?;
75
76 let mut buffer = Vec::with_capacity(usize::try_from(max_bytes).unwrap_or(0));
77 file.read_to_end(&mut buffer)?;
78
79 let content = String::from_utf8_lossy(&buffer);
81
82 tracing::warn!(
84 path = %path.display(),
85 file_size = file_size,
86 max_bytes = max_bytes,
87 "output file truncated: exceeded size limit"
88 );
89
90 Ok(content.into_owned())
91}
92
93pub(crate) fn calculate_backoff_delay(retry_count: usize) -> u64 {
98 let base_delay = config::retry_base_delay_ms();
99 let max_delay = config::retry_max_delay_ms();
100
101 let exponential =
103 base_delay * (2_u64.saturating_pow(u32::try_from(retry_count).unwrap_or(u32::MAX)));
104 let capped = exponential.min(max_delay);
105
106 #[expect(
111 clippy::cast_precision_loss,
112 clippy::cast_possible_truncation,
113 clippy::cast_sign_loss,
114 reason = "a jitter range in milliseconds, far below 2^52 and never negative"
115 )]
116 let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
117 let jitter = {
118 let mut h = DefaultHasher::new();
119 SystemTime::now().hash(&mut h);
120 thread::current().id().hash(&mut h);
121 h.finish() % (jitter_range * 2 + 1)
122 };
123
124 capped.saturating_sub(jitter_range) + jitter
125}
126
127fn fallback_retry_count(task: Option<&Task>) -> usize {
128 task.map_or(0, |t| {
129 let chain_retry_count = t
130 .limit_reroute_chain
131 .split(',')
132 .filter(|entry| !entry.is_empty())
133 .count();
134 let network_retry_count = usize::try_from(t.network_retries).unwrap_or(0);
135 chain_retry_count.max(network_retry_count)
136 })
137}
138
139pub async fn wait_for_fallback_backoff(
142 task_id: &str,
143 store: &Option<Arc<TaskStore>>,
144 repo: &str,
145) -> u64 {
146 let task = store::opt_store_get_task(store, repo, task_id).await;
147
148 let retry_count = fallback_retry_count(task.as_ref());
151
152 let delay = calculate_backoff_delay(retry_count);
153
154 if delay > 0 {
155 tracing::debug!(
156 task_id,
157 retry_count = retry_count,
158 delay_ms = delay,
159 "fallback backoff: waiting before retry"
160 );
161 time::sleep(Duration::from_millis(delay)).await;
162 }
163
164 delay
165}
166
167#[derive(Debug, Clone)]
172pub enum WeightSignal {
173 Success { agent: String },
175 RateLimited { agent: String },
177 Failed { agent: String },
180 Blocked,
182 Rerouted,
185 None,
187}
188
189pub async fn read_output_file(task_id: &str, primary_path: &Path, repo: &str) -> String {
194 if let Some(content) = task::spawn_blocking({
196 let p = primary_path.to_path_buf();
197 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
198 })
199 .await
200 .ok()
201 .and_then(result::Result::ok)
202 && !content.is_empty()
203 {
204 return content;
205 }
206
207 if let Ok(task_dir) = home::task_dir_async(repo, task_id).await {
209 let attempts_dir = task_dir.join("attempts");
210 if tokio_fs::metadata(&attempts_dir)
212 .await
213 .is_ok_and(|m| m.is_dir())
214 {
215 let mut attempt_nums: Vec<u32> = task::spawn_blocking({
216 let a = attempts_dir.clone();
217 move || {
218 match fs::read_dir(&a) {
219 Ok(entries) => {
220 let mut nums = Vec::new();
221 for entry in entries {
222 match entry {
223 Ok(e) => {
224 if let Some(n) =
225 e.file_name().to_str().and_then(|n| n.parse().ok())
226 {
227 nums.push(n);
228 }
229 }
230 Err(e) => tracing::warn!(
231 path = %a.display(),
232 err = %e,
233 "error reading attempt dir entry in output fallback"
234 ),
235 }
236 }
237 nums
238 }
239 Err(e) => {
240 tracing::warn!(path = %a.display(), err = %e, "failed to read attempts dir in output fallback");
241 Vec::new()
242 }
243 }
244 }
245 })
246 .await
247 .unwrap_or_default();
248
249 attempt_nums.sort_unstable_by(|a, b| b.cmp(a)); for n in attempt_nums {
251 let p = attempts_dir.join(n.to_string()).join("output.json");
252 if let Some(content) = task::spawn_blocking({
253 let p = p.clone();
254 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
255 })
256 .await
257 .ok()
258 .and_then(result::Result::ok)
259 && !content.is_empty()
260 {
261 tracing::info!(task_id, path = %p.display(), "read output from attempt dir");
262 return content;
263 }
264 }
265 }
266 }
267
268 let state_dir = home::state_dir().unwrap_or_else(|_| PathBuf::from("/tmp"));
270
271 let mut fallbacks = vec![
272 PathBuf::from(format!("/tmp/output-{task_id}.json")),
273 state_dir.join(format!("output-{task_id}.json")),
274 ];
275
276 if let Ok(legacy_path) = home::state_file(&format!("output-{task_id}.json"))
277 && !fallbacks.contains(&legacy_path)
278 {
279 fallbacks.push(legacy_path);
280 }
281
282 for path in &fallbacks {
283 if let Some(content) = task::spawn_blocking({
284 let p = path.clone();
285 move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
286 })
287 .await
288 .ok()
289 .and_then(result::Result::ok)
290 && !content.is_empty()
291 {
292 tracing::info!(task_id, path = %path.display(), "read output from legacy fallback");
293 return content;
294 }
295 }
296
297 String::new()
298}
299
300pub fn pick_fallback_agent(
302 current_agent: &str,
303 chain: &str,
304 available_agents: &[String],
305) -> Option<String> {
306 clear_expired_cooldowns();
308
309 let chain_set: HashSet<&str> = if chain.is_empty() {
310 HashSet::new()
311 } else {
312 chain.split(',').collect()
313 };
314
315 for agent in available_agents {
319 if agent == current_agent || chain_set.contains(agent.as_str()) {
320 continue;
321 }
322 if is_agent_in_cooldown(agent) {
323 continue;
324 }
325 return Some(agent.clone());
332 }
333
334 None
335}
336
337#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum RetryableError {
340 Timeout,
342 Interrupted,
344 UsageLimit,
346 AuthError,
348 Failed,
350 MissingTooling,
352 ModelUnavailable,
356}
357
358impl RetryableError {
359 pub fn type_str(self) -> &'static str {
362 match self {
363 RetryableError::Timeout => "timeout",
364 RetryableError::Interrupted => "interrupted",
365 RetryableError::UsageLimit => "rate_limit",
366 RetryableError::AuthError => "auth_error",
367 RetryableError::ModelUnavailable => "model_unavailable",
368 RetryableError::Failed | RetryableError::MissingTooling => "failed",
369 }
370 }
371}
372
373#[derive(Debug, Clone, PartialEq, Eq)]
375pub struct FailoverOutcome {
376 pub status: String,
378 pub next_agent: Option<String>,
380}
381
382impl FailoverOutcome {
383 fn with_status(status: impl Into<String>) -> Self {
384 Self {
385 status: status.into(),
386 next_agent: None,
387 }
388 }
389
390 fn switched(next_agent: String) -> Self {
391 Self {
392 status: "new".to_string(),
393 next_agent: Some(next_agent),
394 }
395 }
396}
397
398pub async fn handle_failover(
409 task_id: &str,
410 agent_name: &str,
411 error_type: RetryableError,
412 error_message: &str,
413 credit_recorded: bool,
414 store: &Option<Arc<TaskStore>>,
415 repo: &str,
416) -> FailoverOutcome {
417 let chain = get_reroute_chain(task_id, store, repo).await;
419 let chain = update_reroute_chain(task_id, agent_name, &chain, store, repo).await;
420
421 if error_type == RetryableError::Timeout
422 && let Some(status) =
423 block_after_repeated_timeouts(task_id, error_message, store, repo).await
424 {
425 return FailoverOutcome::with_status(status);
426 }
427
428 let available: Vec<String> = engine::configured_agents()
430 .into_iter()
431 .filter(|a| cmd_cache::command_exists(a))
432 .collect();
433
434 let chain_set: HashSet<&str> = if chain.is_empty() {
436 HashSet::new()
437 } else {
438 chain.split(',').collect()
439 };
440 let is_exhausted = available.iter().all(|a| chain_set.contains(a.as_str()));
441
442 if is_exhausted {
443 let is_retryable = matches!(
449 error_type,
450 RetryableError::UsageLimit
451 | RetryableError::AuthError
452 | RetryableError::ModelUnavailable
453 | RetryableError::Timeout
454 | RetryableError::Interrupted
455 );
456 let next_status = if is_retryable { "new" } else { "blocked" };
457
458 tracing::warn!(
459 task_id,
460 agent = agent_name,
461 error_type = ?error_type,
462 next_status,
463 "all agents exhausted"
464 );
465
466 let mut store_fields: Vec<(&str, serde_json::Value)> = Vec::new();
467
468 if is_retryable {
469 let msg = format!("{error_message} (all agents exhausted, will retry when available)");
472 store_fields.push(("last_error", serde_json::json!(msg)));
473 store_fields.push(("limit_reroute_chain", serde_json::json!("")));
474 } else {
475 let msg = format!("{error_message} (all agents exhausted)");
476 let block_reason = format!(
477 "all agents exhausted: {} — {}",
478 error_type.type_str(),
479 error_message
480 );
481 store_fields.push(("last_error", serde_json::json!(msg)));
482 store_fields.push(("block_reason", serde_json::json!(block_reason)));
483 }
484
485 if let Err(e) = store::store_set_result(store, repo, task_id, &store_fields).await {
486 tracing::warn!(task_id, err = %e, "failed to write agents_exhausted fields to store");
487 }
488
489 let details = serde_json::json!({
491 "failure_reason": error_type.type_str(),
492 "error_message": error_message,
493 "chain": chain,
494 "agents_exhausted": true,
495 "is_retryable": is_retryable,
496 });
497 store::store_log_activity(
498 store,
499 repo,
500 task_id,
501 "rerouted",
502 Some("in_progress"),
503 Some(next_status),
504 None::<&str>,
505 None::<&str>,
506 Some(&details),
507 )
508 .await;
509
510 return FailoverOutcome::with_status(next_status);
511 }
512
513 if let Some(next) = pick_fallback_agent(agent_name, &chain, &available) {
515 tracing::info!(
516 task_id,
517 from = agent_name,
518 to = next,
519 error_type = ?error_type,
520 "failover: switching to fallback agent"
521 );
522
523 if !matches!(
525 error_type,
526 RetryableError::MissingTooling
527 | RetryableError::ModelUnavailable
528 | RetryableError::Timeout
529 | RetryableError::Interrupted
530 ) {
531 if !credit_recorded {
534 record_agent_failure_with_message(agent_name, error_message).await;
535 }
536
537 set_agent_cooldown(agent_name, 120).await;
542 }
543
544 let msg = format!("{error_message}, switching to {next} for re-route");
547 if let Err(e) = store::store_set_result(
548 store,
549 repo,
550 task_id,
551 &[
552 ("agent", serde_json::json!(next)),
553 ("model", serde_json::json!("")),
554 ("last_error", serde_json::json!(msg)),
555 ],
556 )
557 .await
558 {
559 tracing::warn!(task_id, err = %e, "failed to write failover agent to store");
560 }
561
562 wait_for_fallback_backoff(task_id, store, repo).await;
568
569 let details = serde_json::json!({
571 "failure_reason": error_type.type_str(),
572 "error_message": error_message,
573 "from_agent": agent_name,
574 "to_agent": next,
575 "chain": chain,
576 });
577 store::store_log_activity(
578 store,
579 repo,
580 task_id,
581 "rerouted",
582 Some("in_progress"),
583 Some("new"),
584 Some(&next),
585 None::<&str>,
586 Some(&details),
587 )
588 .await;
589
590 return FailoverOutcome::switched(next);
591 }
592
593 tracing::warn!(task_id, agent = agent_name, "no fallback agents available");
595
596 let is_retryable = matches!(
597 error_type,
598 RetryableError::UsageLimit
599 | RetryableError::AuthError
600 | RetryableError::ModelUnavailable
601 | RetryableError::Timeout
602 | RetryableError::Interrupted
603 );
604
605 if !matches!(
607 error_type,
608 RetryableError::MissingTooling
609 | RetryableError::ModelUnavailable
610 | RetryableError::Timeout
611 | RetryableError::Interrupted
612 ) && !credit_recorded
613 {
614 record_agent_failure_with_message(agent_name, error_message).await;
615 }
616
617 if is_retryable {
618 wait_for_fallback_backoff(task_id, store, repo).await;
621
622 let msg = format!("{error_message} (no fallback agents, will retry)");
623 if let Err(e) = store::store_set_result(
624 store,
625 repo,
626 task_id,
627 &[
628 ("last_error", serde_json::json!(msg)),
629 ("limit_reroute_chain", serde_json::json!("")),
630 ],
631 )
632 .await
633 {
634 tracing::warn!(task_id, err = %e, "failed to write no_fallback retry fields to store");
635 }
636
637 let details = serde_json::json!({
638 "failure_reason": error_type.type_str(),
639 "error_message": error_message,
640 "chain": chain,
641 "no_fallback_available": true,
642 "is_retryable": true,
643 });
644 store::store_log_activity(
645 store,
646 repo,
647 task_id,
648 "rerouted",
649 Some("in_progress"),
650 Some("new"),
651 None::<&str>,
652 None::<&str>,
653 Some(&details),
654 )
655 .await;
656
657 return FailoverOutcome::with_status("new");
658 }
659
660 let msg = format!("{error_message}, no fallback agents");
661 if let Err(e) = store::store_set_result(
662 store,
663 repo,
664 task_id,
665 &[("last_error", serde_json::json!(msg))],
666 )
667 .await
668 {
669 tracing::warn!(task_id, err = %e, "failed to write no_fallback last_error to store");
670 }
671
672 let details = serde_json::json!({
674 "failure_reason": error_type.type_str(),
675 "error_message": error_message,
676 "chain": chain,
677 "no_fallback_available": true,
678 });
679 store::store_log_activity(
680 store,
681 repo,
682 task_id,
683 "rerouted",
684 Some("in_progress"),
685 Some("needs_review"),
686 None::<&str>,
687 None::<&str>,
688 Some(&details),
689 )
690 .await;
691
692 FailoverOutcome::with_status("needs_review")
693}
694
695pub async fn block_after_repeated_timeouts(
698 task_id: &str,
699 error_message: &str,
700 store: &Option<Arc<TaskStore>>,
701 repo: &str,
702) -> Option<String> {
703 let max: i64 = app_config::get("workflow.max_timeout_retries")
704 .ok()
705 .and_then(|v| v.parse().ok())
706 .unwrap_or(3);
707 let count = match store::store_increment(store, repo, task_id, "timeout_retries").await {
708 Ok(count) => i64::try_from(count).unwrap_or(i64::MAX),
709 Err(e) => {
710 tracing::warn!(task_id, err = %e, "failed to increment timeout_retries");
711 return None;
712 }
713 };
714 if count < max {
715 return None;
716 }
717
718 let block_reason = format!("timed out {count} times");
719 tracing::warn!(
720 task_id,
721 count,
722 max,
723 "timeout retries exhausted, blocking task"
724 );
725 if let Err(e) = store::store_set_result(
726 store,
727 repo,
728 task_id,
729 &[
730 (
731 "last_error",
732 serde_json::json!(format!("{error_message} ({block_reason})")),
733 ),
734 ("block_reason", serde_json::json!(block_reason)),
735 ("limit_reroute_chain", serde_json::json!("")),
736 ],
737 )
738 .await
739 {
740 tracing::warn!(task_id, err = %e, "failed to write timeout block fields to store");
741 }
742 let details = serde_json::json!({
743 "failure_reason": "timeout",
744 "error_message": error_message,
745 "timeout_retries": count,
746 });
747 store::store_log_activity(
748 store,
749 repo,
750 task_id,
751 "rerouted",
752 Some("in_progress"),
753 Some("blocked"),
754 None::<&str>,
755 None::<&str>,
756 Some(&details),
757 )
758 .await;
759 Some("blocked".to_string())
760}
761
762pub async fn get_reroute_chain(
764 task_id: &str,
765 store: &Option<Arc<TaskStore>>,
766 repo: &str,
767) -> String {
768 store::opt_store_get_task(store, repo, task_id)
769 .await
770 .map(|t| t.limit_reroute_chain)
771 .unwrap_or_default()
772 .trim()
773 .to_string()
774}
775
776pub async fn update_reroute_chain(
778 task_id: &str,
779 current_agent: &str,
780 existing_chain: &str,
781 store: &Option<Arc<TaskStore>>,
782 repo: &str,
783) -> String {
784 let mut chain = existing_chain.to_string();
785 if chain.is_empty() {
786 chain = current_agent.to_string();
787 } else if !chain.split(',').any(|a| a == current_agent) {
788 chain = format!("{chain},{current_agent}");
789 }
790
791 if let Err(e) = store::store_set_result(
792 store,
793 repo,
794 task_id,
795 &[("limit_reroute_chain", serde_json::json!(chain))],
796 )
797 .await
798 {
799 tracing::warn!(task_id, err = %e, "failed to write limit_reroute_chain to store");
800 }
801 chain
802}
803
804#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
806pub struct ReviewResponse {
807 pub decision: String,
809 pub notes: String,
811 pub test_results: Option<String>,
813 pub issues: Vec<ReviewIssue>,
815}
816
817#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
819pub struct ReviewIssue {
820 pub file: String,
822 pub line: Option<u32>,
824 pub severity: String,
826 pub description: String,
828}
829
830pub fn parse_review_response(text: &str) -> anyhow::Result<ReviewResponse> {
836 let resp = parse_review_json(text)?;
837 if matches!(resp.decision.as_str(), "approve" | "request_changes") {
838 return Ok(resp);
839 }
840
841 let stripped = text.replace("approve|request_changes", "");
844 infer_review_response_from_text(&stripped)
845 .ok_or_else(|| anyhow::anyhow!("invalid review decision: {}", resp.decision))
846}
847
848fn parse_review_json(text: &str) -> anyhow::Result<ReviewResponse> {
849 if let Ok(resp) = serde_json::from_str::<ReviewResponse>(text) {
851 return Ok(resp);
852 }
853
854 if let Some(json_str) = extract_json_block(text)
856 && let Ok(resp) = serde_json::from_str::<ReviewResponse>(&json_str)
857 {
858 return Ok(resp);
859 }
860
861 if let Some(resp) = extract_review_response_object(text) {
862 return Ok(resp);
863 }
864
865 anyhow::bail!("failed to parse review response")
866}
867
868fn infer_review_response_from_text(text: &str) -> Option<ReviewResponse> {
869 let lower = text.to_ascii_lowercase();
870 let has_changes_requested =
871 lower.contains("changes requested") || lower.contains("request_changes");
872 if has_changes_requested {
873 return Some(ReviewResponse {
874 decision: "request_changes".to_string(),
875 notes: "Inferred decision from plain-text review output.".to_string(),
876 test_results: None,
877 issues: Vec::new(),
878 });
879 }
880
881 let positive_approval = lower
884 .split(|c: char| !c.is_ascii_alphabetic())
885 .any(|word| word == "lgtm")
886 || lower.contains("decision is `approve`")
887 || lower.contains("decision: `approve`")
888 || lower.contains("decision is approve")
889 || lower.contains("decision: approve");
890 if positive_approval {
891 return Some(ReviewResponse {
892 decision: "approve".to_string(),
893 notes: "Inferred decision from plain-text review output.".to_string(),
894 test_results: None,
895 issues: Vec::new(),
896 });
897 }
898
899 None
900}
901
902pub fn infer_review_response(text: &str) -> Option<ReviewResponse> {
906 infer_review_response_from_text(text)
907}
908
909fn extract_json_block(text: &str) -> Option<String> {
914 let mut search_from = 0;
915 while let Some(start) = text[search_from..].find("```json") {
916 let abs_start = search_from + start;
917 let after_tag = abs_start + "```json".len();
918 let Some(rest) = text[after_tag..].strip_prefix('\n') else {
919 search_from = after_tag;
920 continue;
921 };
922 let content_start = text.len() - rest.len();
923 let Some(end_offset) = rest.find("```") else {
924 search_from = content_start;
925 continue;
926 };
927 let end = content_start + end_offset;
928 let content = text[content_start..end].trim();
929 if content.starts_with('{') {
930 return Some(content.to_string());
931 }
932 search_from = end + 3;
933 }
934 None
935}
936
937fn extract_review_response_object(text: &str) -> Option<ReviewResponse> {
941 let mut start: Option<usize> = None;
942 let mut depth: i32 = 0;
943 let mut in_string = false;
944 let mut escape = false;
945
946 for (idx, ch) in text.char_indices() {
947 if start.is_some() {
948 if in_string {
949 if escape {
950 escape = false;
951 } else if ch == '\\' {
952 escape = true;
953 } else if ch == '"' {
954 in_string = false;
955 }
956 continue;
957 }
958
959 match ch {
960 '"' => in_string = true,
961 '{' => depth += 1,
962 '}' => {
963 depth -= 1;
964 if depth == 0 {
965 debug_assert!(
966 start.is_some(),
967 "start must be Some when depth returns to 0"
968 );
969 let start_idx = start.unwrap_or(0);
970 let candidate = &text[start_idx..=idx];
971 if let Ok(resp) = serde_json::from_str::<ReviewResponse>(candidate) {
972 return Some(resp);
973 }
974 start = None;
975 }
976 }
977 _ => {}
978 }
979 continue;
980 }
981
982 if ch == '{' {
983 start = Some(idx);
984 depth = 1;
985 in_string = false;
986 escape = false;
987 }
988 }
989
990 None
991}
992
993#[expect(
995 clippy::too_many_arguments,
996 reason = "a memory entry records the task, attempt, agent, model and outcome"
997)]
998pub async fn store_learnings_from_response(
999 task_id: &str,
1000 attempt: u32,
1001 agent: &str,
1002 model: Option<&str>,
1003 response: &AgentResponse,
1004 error: Option<&str>,
1005 store: &Option<Arc<TaskStore>>,
1006 repo: &str,
1007 raw_stdout: &str,
1008) {
1009 let files_modified = if response.files.is_empty() {
1011 let extracted = ndjson::extract_files_changed(raw_stdout);
1012 if !extracted.is_empty() {
1013 tracing::debug!(
1014 task_id,
1015 count = extracted.len(),
1016 "files_modified: fell back to NDJSON extraction (agent reported none)"
1017 );
1018 }
1019 extracted
1020 } else {
1021 response.files.clone()
1022 };
1023
1024 let entry = MemoryEntry {
1026 attempt,
1027 agent: agent.to_string(),
1028 model: model.map(String::from),
1029 learnings: response.learnings.clone(),
1030 error: error.map(String::from),
1031 files_modified,
1032 approach: response.summary.clone(),
1033 timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1034 };
1035
1036 if let Some(st) = store
1037 && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1038 {
1039 match st.append_memory(store_id, &entry).await {
1040 Err(e) => {
1041 tracing::warn!(task_id, error = ?e, "failed to store memory");
1042 }
1043 _ => {
1044 tracing::debug!(task_id, attempt, "stored memory for attempt");
1045 }
1046 }
1047 }
1048}
1049
1050pub async fn store_failure_memory(
1052 task_id: &str,
1053 attempt: u32,
1054 agent: &str,
1055 model: Option<&str>,
1056 error: &str,
1057 store: &Option<Arc<TaskStore>>,
1058 repo: &str,
1059) {
1060 let entry = MemoryEntry {
1061 attempt,
1062 agent: agent.to_string(),
1063 model: model.map(String::from),
1064 learnings: vec![],
1065 error: Some(error.to_string()),
1066 files_modified: vec![],
1067 approach: String::new(),
1068 timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1069 };
1070
1071 if let Some(st) = store
1072 && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1073 {
1074 match st.append_memory(store_id, &entry).await {
1075 Err(e) => {
1076 tracing::warn!(task_id, error = ?e, "failed to store failure memory");
1077 }
1078 _ => {
1079 tracing::debug!(task_id, attempt, "stored failure memory");
1080 }
1081 }
1082 }
1083}
1084
1085#[cfg(test)]
1086mod tests {
1087 use std::env;
1088 #[cfg(unix)]
1089 use std::os::unix::fs::PermissionsExt;
1090 use std::process;
1091 use std::sync::Arc;
1092
1093 use super::super::TaskRunner;
1094 use super::*;
1095 use crate::backends::{ExternalId, ExternalTask};
1096 use crate::engine::cooldown;
1097 use crate::engine::router::{Router, RouterConfig};
1098 use crate::engine::runner::agents::{AgentError, find_agent_result, patterns};
1099 use crate::store::TaskStatus;
1100 use serial_test::serial;
1101
1102 #[serial(cooldown_state)]
1103 #[test]
1104 fn calculate_backoff_delay_jitter_is_centered() {
1105 let base = config::retry_base_delay_ms();
1109 let max = config::retry_max_delay_ms();
1110
1111 let capped = base.min(max);
1113 #[expect(
1114 clippy::cast_precision_loss,
1115 clippy::cast_possible_truncation,
1116 clippy::cast_sign_loss,
1117 reason = "mirrors the jitter range in calculate_backoff_delay"
1118 )]
1119 let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
1120 let lo = capped.saturating_sub(jitter_range);
1121 let hi = capped + jitter_range;
1122
1123 for _ in 0..200 {
1124 let delay = calculate_backoff_delay(0);
1125 assert!(
1126 delay >= lo && delay <= hi,
1127 "delay {delay} out of expected range [{lo}, {hi}]"
1128 );
1129 }
1130 }
1131
1132 #[serial(cooldown_state)]
1133 #[test]
1134 fn fallback_retry_count_uses_larger_network_retry_streak() {
1135 let task = Task {
1136 id: 1,
1137 external_id: None,
1138 repo: "owner/repo".to_string(),
1139 origin: "internal".to_string(),
1140 title: "test".to_string(),
1141 body: String::new(),
1142 status: TaskStatus::New,
1143 source: String::new(),
1144 source_id: String::new(),
1145 author: String::new(),
1146 url: String::new(),
1147 labels: vec![],
1148 agent: None,
1149 model: None,
1150 complexity: "simple".to_string(),
1151 estimate: 0,
1152 route_reason: String::new(),
1153 agent_profile: String::new(),
1154 selected_skills: String::new(),
1155 route_attempts: 0,
1156 attempts: 0,
1157 branch: String::new(),
1158 worktree: String::new(),
1159 worktree_cleaned: false,
1160 owner_node: None,
1161 summary: String::new(),
1162 last_error: String::new(),
1163 parent_id: None,
1164 block_reason: None,
1165 pr_number: None,
1166 pr_review_context: String::new(),
1167 last_review_ts: String::new(),
1168 review_ts_map: "{}".to_string(),
1169 last_comment_review_ts: String::new(),
1170 merge_conflict_retries: 0,
1171 ci_merge_failures: 0,
1172 pr_create_failures: 0,
1173 push_failures: 0,
1174 network_retries: 3,
1175 timeout_retries: 0,
1176 review_agent_failures: 0,
1177 review_resets: 0,
1178 review_cycles: 0,
1179 review_invocations: 0,
1180 needs_review_refires: 0,
1181 input_tokens: 0,
1182 output_tokens: 0,
1183 total_cost_usd: 0.0,
1184 model_reroute_chain: String::new(),
1185 limit_reroute_chain: "claude".to_string(),
1186 memory: vec![],
1187 delegations: vec![],
1188 auto_unblock_count: 0,
1189 auto_unblock_last_at: String::new(),
1190 auto_unblock_last_reason: String::new(),
1191 ci_recovery_count: 0,
1192 no_code_reroutes: 0,
1193 no_code_last_agent: String::new(),
1194 created_at: String::new(),
1195 updated_at: String::new(),
1196 };
1197
1198 assert_eq!(fallback_retry_count(Some(&task)), 3);
1199 }
1200
1201 #[serial(cooldown_state)]
1202 #[test]
1203 fn retryable_type_str_returns_correct_labels() {
1204 assert_eq!(RetryableError::Timeout.type_str(), "timeout");
1205 assert_eq!(RetryableError::Interrupted.type_str(), "interrupted");
1206 assert_eq!(RetryableError::UsageLimit.type_str(), "rate_limit");
1207 assert_eq!(RetryableError::AuthError.type_str(), "auth_error");
1208 assert_eq!(RetryableError::Failed.type_str(), "failed");
1209 assert_eq!(RetryableError::MissingTooling.type_str(), "failed");
1210 }
1211
1212 #[serial(cooldown_state)]
1213 #[test]
1214 fn is_usage_limit_detects_rate_limit() {
1215 assert!(patterns::detect_rate_limit("Error: rate limit exceeded").is_some());
1216 assert!(patterns::detect_rate_limit("HTTP 429 Too Many Requests").is_some());
1217 assert!(patterns::detect_rate_limit("quota exceeded for model").is_some());
1218 assert!(patterns::detect_context_overflow("context_length_exceeded").is_some());
1220 }
1221
1222 #[serial(cooldown_state)]
1223 #[test]
1224 fn is_usage_limit_rejects_normal_text() {
1225 assert!(patterns::detect_rate_limit("task completed successfully").is_none());
1226 assert!(patterns::detect_rate_limit("").is_none());
1227 }
1228
1229 #[serial(cooldown_state)]
1230 #[test]
1231 fn is_auth_error_detects_common_patterns() {
1232 assert!(patterns::detect_auth_error("401 Unauthorized").is_some());
1233 assert!(patterns::detect_auth_error("invalid api key provided").is_some());
1234 assert!(patterns::detect_auth_error("Your billing plan has expired").is_some());
1235 assert!(patterns::detect_auth_error("Error: 403 Forbidden").is_some());
1236 }
1237
1238 #[serial(cooldown_state)]
1239 #[test]
1240 fn is_auth_error_rejects_normal_text() {
1241 assert!(patterns::detect_auth_error("task completed successfully").is_none());
1242 assert!(patterns::detect_auth_error("").is_none());
1243 }
1244
1245 #[serial(cooldown_state)]
1246 #[test]
1247 fn detect_missing_tooling_finds_known_tools() {
1248 let result = patterns::detect_missing_tool("bun: command not found");
1249 assert!(result.is_some());
1250 if let AgentError::MissingTool { tool } = result.unwrap() {
1251 assert_eq!(tool, "bun");
1252 } else {
1253 panic!("expected MissingTool error");
1254 }
1255
1256 let result = patterns::detect_missing_tool("env: anchor: no such file");
1257 assert!(result.is_some());
1258 if let AgentError::MissingTool { tool } = result.unwrap() {
1259 assert_eq!(tool, "anchor");
1260 } else {
1261 panic!("expected MissingTool error");
1262 }
1263
1264 let result = patterns::detect_missing_tool("spawn docker enoent");
1265 assert!(result.is_some());
1266 if let AgentError::MissingTool { tool } = result.unwrap() {
1267 assert_eq!(tool, "docker");
1268 } else {
1269 panic!("expected MissingTool error");
1270 }
1271 }
1272
1273 #[serial(cooldown_state)]
1274 #[test]
1275 fn detect_missing_tooling_returns_none_for_normal() {
1276 assert!(patterns::detect_missing_tool("everything works fine").is_none());
1277 assert!(patterns::detect_missing_tool("").is_none());
1278 }
1279
1280 #[serial(cooldown_state)]
1283 #[tokio::test]
1284 async fn record_and_check_agent_cooldown() {
1285 cooldown::reset_global_state().await;
1286 let agent = "test_cooldown_agent_1";
1288 assert!(!is_agent_in_cooldown(agent));
1289 record_agent_failure_with_message(agent, "").await;
1290 assert!(is_agent_in_cooldown(agent));
1291 }
1292
1293 #[serial(cooldown_state)]
1294 #[tokio::test]
1295 async fn record_and_check_model_cooldown() {
1296 cooldown::reset_global_state().await;
1297 let agent = "test_cooldown_agent_2";
1298 let model = "test_model_x";
1299 assert!(!is_model_in_cooldown(agent, model));
1300 record_model_failure(agent, model).await;
1301 assert!(is_model_in_cooldown(agent, model));
1302 assert!(!is_model_in_cooldown(agent, "other_model"));
1304 }
1305
1306 #[serial(cooldown_state)]
1307 #[tokio::test]
1308 async fn clear_expired_does_not_remove_fresh_entries() {
1309 cooldown::reset_global_state().await;
1310 let agent = "test_cooldown_agent_3";
1311 record_agent_failure_with_message(agent, "").await;
1312 clear_expired_cooldowns();
1313 assert!(is_agent_in_cooldown(agent));
1315 }
1316
1317 #[serial(cooldown_state)]
1319 #[test]
1320 fn pick_fallback_skips_current_agent() {
1321 let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1322 let result = pick_fallback_agent("test_agent_a", "", &available);
1323 assert_eq!(result, Some("test_agent_b".to_string()));
1324 }
1325
1326 #[serial(cooldown_state)]
1327 #[test]
1328 fn pick_fallback_skips_chain_agents() {
1329 let available = vec![
1330 "test_agent_a".to_string(),
1331 "test_agent_b".to_string(),
1332 "test_agent_c".to_string(),
1333 ];
1334 let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1335 assert_eq!(result, Some("test_agent_c".to_string()));
1336 }
1337
1338 #[serial(cooldown_state)]
1339 #[test]
1340 fn pick_fallback_returns_none_when_exhausted() {
1341 let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1342 let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1343 assert!(result.is_none());
1344 }
1345
1346 #[serial(cooldown_state)]
1347 #[test]
1348 fn weight_signal_variants() {
1349 let success = WeightSignal::Success {
1350 agent: "claude".to_string(),
1351 };
1352 let limited = WeightSignal::RateLimited {
1353 agent: "codex".to_string(),
1354 };
1355 let none = WeightSignal::None;
1356
1357 assert!(format!("{success:?}").contains("claude"));
1359 assert!(format!("{limited:?}").contains("codex"));
1360 assert!(format!("{none:?}").contains("None"));
1361 }
1362
1363 #[serial(cooldown_state)]
1364 #[test]
1365 fn parse_review_response_direct_json() {
1366 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1367 let resp = parse_review_response(json).unwrap();
1368 assert_eq!(resp.decision, "approve");
1369 assert_eq!(resp.notes, "LGTM");
1370 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1371 }
1372
1373 #[serial(cooldown_state)]
1374 #[test]
1375 fn parse_review_response_from_markdown() {
1376 let md = r#"Here is my review:
1377
1378```json
1379{"decision":"request_changes","notes":"Fix the bug","issues":[{"file":"src/main.rs","line":10,"severity":"error","description":"null deref"}]}
1380```
1381
1382That's all."#;
1383 let resp = parse_review_response(md).unwrap();
1384 assert_eq!(resp.decision, "request_changes");
1385 assert_eq!(resp.issues.len(), 1);
1386 assert_eq!(resp.issues[0].file, "src/main.rs");
1387 }
1388
1389 #[serial(cooldown_state)]
1390 #[test]
1391 fn parse_review_response_embedded_json_object() {
1392 let text = "Review complete.\n\n{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}\nThanks!";
1393 let resp = parse_review_response(text).unwrap();
1394 assert_eq!(resp.decision, "approve");
1395 assert_eq!(resp.notes, "LGTM");
1396 }
1397
1398 #[serial(cooldown_state)]
1399 #[test]
1400 fn parse_review_response_template_echo_infers_from_notes() {
1401 let json = r#"{"decision":"approve|request_changes","notes":"LGTM","issues":[]}"#;
1402 assert_eq!(parse_review_response(json).unwrap().decision, "approve");
1403
1404 let no_signal = r#"{"decision":"approve|request_changes","notes":"hmm","issues":[]}"#;
1405 assert!(parse_review_response(no_signal).is_err());
1406 }
1407
1408 #[serial(cooldown_state)]
1409 #[test]
1410 fn parse_review_response_invalid() {
1411 let result = parse_review_response("not json at all");
1412 assert!(result.is_err());
1413 }
1414
1415 #[serial(cooldown_state)]
1416 #[test]
1417 fn extract_json_block_basic() {
1418 let md = "text\n```json\n{\"key\": \"value\"}\n```\nmore";
1419 let result = extract_json_block(md);
1420 assert!(result.is_some());
1421 assert!(result.unwrap().contains("\"key\""));
1422 }
1423
1424 #[serial(cooldown_state)]
1425 #[test]
1426 fn extract_json_block_skips_non_json_blocks() {
1427 let md = "```json\nnot-json-array\n```\n\n```json\n{\"real\": true}\n```";
1428 let result = extract_json_block(md);
1429 assert!(result.is_some());
1430 assert!(result.unwrap().contains("\"real\""));
1431 }
1432
1433 #[serial(cooldown_state)]
1434 #[test]
1435 fn extract_json_block_none_when_missing() {
1436 assert!(extract_json_block("no code blocks here").is_none());
1437 }
1438
1439 #[serial(cooldown_state)]
1440 #[test]
1441 fn extract_json_block_skips_malformed_intermediate_blocks() {
1442 let md = "prefix ```json{\"broken\": true}\n\n```json\n{\"real\": true}\n```";
1443 let result = extract_json_block(md);
1444 assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1445 }
1446
1447 #[serial(cooldown_state)]
1448 #[test]
1449 fn extract_json_block_skips_multiple_malformed_intermediate_blocks() {
1450 let md = "prefix ```json{\"broken\": true}\nmid ```json[1, 2, 3]\n\n```json\n{\"real\": true}\n```";
1451 let result = extract_json_block(md);
1452 assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1453 }
1454
1455 fn parse_review_plain(text: &str) -> anyhow::Result<ReviewResponse> {
1460 parse_review_response(text).or_else(|_| {
1461 infer_review_response(text)
1462 .ok_or_else(|| anyhow::anyhow!("failed to parse review response"))
1463 })
1464 }
1465
1466 #[serial(cooldown_state)]
1467 #[test]
1468 fn parse_review_from_output_direct_json() {
1469 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1470 let resp = parse_review_response(json).unwrap();
1471 assert_eq!(resp.decision, "approve");
1472 }
1473
1474 #[serial(cooldown_state)]
1475 #[test]
1476 fn parse_review_from_output_markdown() {
1477 let md = "Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}\n```\n";
1478 let resp = parse_review_response(md).unwrap();
1479 assert_eq!(resp.decision, "request_changes");
1480 }
1481
1482 #[serial(cooldown_state)]
1483 #[test]
1484 fn parse_review_from_output_plain_text_fallback() {
1485 let text = "The review is done. LGTM.";
1486 let resp = parse_review_plain(text).unwrap();
1487 assert_eq!(resp.decision, "approve");
1488 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1489 }
1490
1491 #[serial(cooldown_state)]
1493 #[test]
1494 fn parse_review_from_output_ndjson_part_format() {
1495 let ndjson = concat!(
1496 r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1497 "\n",
1498 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Reviewing the PR...\n\n```json\n{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1499 "\n",
1500 r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop"}}"#,
1501 );
1502 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1503 assert_eq!(resp.decision, "approve");
1504 assert_eq!(resp.notes, "All checks pass");
1505 }
1506
1507 #[serial(cooldown_state)]
1509 #[test]
1510 fn parse_review_from_output_ndjson_direct_text_format() {
1511 let ndjson = concat!(
1512 r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1513 "\n",
1514 r#"{"type":"text","timestamp":1001,"sessionID":"ses_abc","text":"Here is my review:\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the bug\",\"test_results\":\"fail\",\"issues\":[]}\n```\n"}"#,
1515 "\n",
1516 r#"{"type":"step_finish","timestamp":1002,"sessionID":"ses_abc"}"#,
1517 );
1518 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1519 assert_eq!(resp.decision, "request_changes");
1520 assert_eq!(resp.notes, "Fix the bug");
1521 }
1522
1523 #[serial(cooldown_state)]
1525 #[test]
1526 fn parse_review_from_output_ndjson_direct_json_in_text_event() {
1527 let ndjson = concat!(
1528 r#"{"type":"step_start","timestamp":1000}"#,
1529 "\n",
1530 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1531 "\n",
1532 r#"{"type":"step_finish","timestamp":1002}"#,
1533 );
1534 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1535 assert_eq!(resp.decision, "approve");
1536 }
1537
1538 #[serial(cooldown_state)]
1539 #[test]
1540 fn parse_review_from_output_ndjson_prefers_final_json_text_event() {
1541 let ndjson = concat!(
1542 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Working on it..."}}"#,
1543 "\n",
1544 r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1545 );
1546 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1547 assert_eq!(resp.decision, "approve");
1548 assert_eq!(resp.notes, "LGTM");
1549 }
1550
1551 #[serial(cooldown_state)]
1553 #[test]
1554 fn parse_review_from_output_ndjson_concatenated_text() {
1555 let ndjson = concat!(
1556 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Here is my\n"}}"#,
1557 "\n",
1558 r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"review:\n\n```json\n{\"decision\":\"approve\",\"notes\":\"OK\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1559 );
1560 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1561 assert_eq!(resp.decision, "approve");
1562 }
1563
1564 #[serial(cooldown_state)]
1569 #[test]
1570 fn parse_review_from_output_codex_agent_message() {
1571 let ndjson = concat!(
1572 r#"{"type":"thread.started","thread_id":"t1"}"#,
1573 "\n",
1574 r#"{"type":"turn.started"}"#,
1575 "\n",
1576 r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Planning git commands..."}}"#,
1577 "\n",
1578 r#"{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"git log --oneline -5","output":"abc123 fix: something"}}"#,
1579 "\n",
1580 r#"{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1581 "\n",
1582 r#"{"type":"turn.completed"}"#,
1583 );
1584 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1585 assert_eq!(resp.decision, "approve");
1586 assert_eq!(resp.notes, "All checks pass");
1587 }
1588
1589 #[serial(cooldown_state)]
1591 #[test]
1592 fn parse_review_from_output_codex_agent_message_markdown() {
1593 let ndjson = concat!(
1594 r#"{"type":"thread.started","thread_id":"t1"}"#,
1595 "\n",
1596 r#"{"type":"turn.started"}"#,
1597 "\n",
1598 r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Reviewing..."}}"#,
1599 "\n",
1600 r#"{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the tests\",\"issues\":[]}\n```\n"}}"#,
1601 "\n",
1602 r#"{"type":"turn.completed"}"#,
1603 );
1604 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1605 assert_eq!(resp.decision, "request_changes");
1606 assert_eq!(resp.notes, "Fix the tests");
1607 }
1608
1609 #[serial(cooldown_state)]
1611 #[test]
1612 fn codex_skips_reasoning_items() {
1613 let ndjson = concat!(
1614 r#"{"type":"item.completed","item":{"type":"reasoning","text":"internal thoughts..."}}"#,
1615 "\n",
1616 r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}}"#,
1617 );
1618 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1619 assert_eq!(resp.decision, "approve");
1620 assert_eq!(resp.notes, "LGTM");
1621 }
1622
1623 #[serial(cooldown_state)]
1625 #[test]
1626 fn infer_review_response_negation_not_approved() {
1627 assert!(
1628 parse_review_plain("The changes were not approved.").is_err(),
1629 "\"not approved\" should not infer approval"
1630 );
1631 }
1632
1633 #[serial(cooldown_state)]
1634 #[test]
1635 fn infer_review_response_negation_unapproved() {
1636 assert!(
1637 parse_review_plain("Unapproved changes found in the diff.").is_err(),
1638 "\"unapproved\" should not infer approval"
1639 );
1640 }
1641
1642 #[serial(cooldown_state)]
1643 #[test]
1644 fn infer_review_response_negation_not_approving() {
1645 assert!(
1646 parse_review_plain("I am not approving this PR.").is_err(),
1647 "\"not approving\" should not infer approval"
1648 );
1649 }
1650
1651 #[serial(cooldown_state)]
1652 #[test]
1653 fn infer_review_response_negation_cannot_be_approved() {
1654 assert!(
1655 parse_review_plain("This PR cannot be approved until the race in tick.rs is fixed.")
1656 .is_err(),
1657 "\"cannot be approved\" should not infer approval"
1658 );
1659 }
1660
1661 #[serial(cooldown_state)]
1662 #[test]
1663 fn infer_review_response_negation_should_not_be_approved() {
1664 assert!(
1665 parse_review_plain("The diff breaks the build and should not be approved.").is_err(),
1666 "\"should not be approved\" should not infer approval"
1667 );
1668 }
1669
1670 #[serial(cooldown_state)]
1671 #[test]
1672 fn infer_review_response_negation_not_yet_approved() {
1673 assert!(
1674 parse_review_plain("This change is not yet approved.").is_err(),
1675 "\"not yet approved\" should not infer approval"
1676 );
1677 }
1678
1679 #[serial(cooldown_state)]
1680 #[test]
1681 fn infer_review_response_negation_wont_approve() {
1682 assert!(
1683 parse_review_plain("I won't approve this until tests cover the retry path.").is_err(),
1684 "\"won't approve\" should not infer approval"
1685 );
1686 }
1687
1688 #[serial(cooldown_state)]
1689 #[test]
1690 fn infer_review_response_bare_approved() {
1691 assert!(
1692 parse_review_plain("The PR was approved.").is_err(),
1693 "bare \"approved\" should not infer approval"
1694 );
1695 }
1696
1697 #[serial(cooldown_state)]
1699 #[test]
1700 fn infer_review_response_lgtm() {
1701 let resp = parse_review_plain("LGTM, everything looks fine.").unwrap();
1702 assert_eq!(resp.decision, "approve");
1703 }
1704
1705 #[serial(cooldown_state)]
1707 #[test]
1708 fn infer_review_response_looks_good() {
1709 assert!(
1710 parse_review_plain("Looks good to me!").is_err(),
1711 "\"looks good\" should not infer approval"
1712 );
1713 }
1714
1715 #[serial(cooldown_state)]
1716 #[test]
1717 fn infer_review_response_all_checks_passed() {
1718 assert!(
1719 parse_review_plain(
1720 "The background task completed but I already got the CI results directly. All checks passed — no action needed.",
1721 )
1722 .is_err(),
1723 "\"all checks passed\" should not infer approval"
1724 );
1725 }
1726
1727 #[serial(cooldown_state)]
1728 #[test]
1729 fn infer_review_response_checks_passed() {
1730 assert!(
1731 parse_review_plain("All checks passed (fmt ✓, clippy ✓, tests ✓).").is_err(),
1732 "\"all checks passed\" should not infer approval"
1733 );
1734 }
1735
1736 #[serial(cooldown_state)]
1737 #[test]
1738 fn infer_review_response_all_tests_passed() {
1739 assert!(
1740 parse_review_plain("All tests passed and the implementation looks correct.").is_err(),
1741 "\"all tests passed\" should not infer approval"
1742 );
1743 }
1744
1745 #[serial(cooldown_state)]
1746 #[test]
1747 fn infer_review_response_no_issues_found() {
1748 assert!(
1749 parse_review_plain("Review complete. No issues found in the diff.").is_err(),
1750 "\"no issues found\" should not infer approval"
1751 );
1752 }
1753
1754 #[serial(cooldown_state)]
1756 #[test]
1757 fn infer_review_response_partial_tests_passed_no_approve() {
1758 let text = "The existing tests passed before this change but the new payment logic has a null-pointer risk on line 42.";
1759 assert!(parse_review_plain(text).is_err());
1760 }
1761
1762 #[serial(cooldown_state)]
1764 #[test]
1765 fn infer_review_response_partial_checks_passed_no_approve() {
1766 let text = "CI checks passed for the base branch but this PR adds a broken path.";
1767 assert!(parse_review_plain(text).is_err());
1768 }
1769
1770 #[serial(cooldown_state)]
1772 #[test]
1773 fn infer_review_response_decision_is_approve_backtick() {
1774 let text = "Already retrieved the output and completed the review above. The CI test check passed (3m11s) and all other checks are green. Decision is `approve`.";
1775 let resp = parse_review_plain(text).unwrap();
1776 assert_eq!(resp.decision, "approve");
1777 }
1778
1779 #[serial(cooldown_state)]
1781 #[test]
1782 fn infer_review_response_decision_colon_approve() {
1783 let text = "Reviewed the PR. No blocking issues found. Decision: approve";
1784 let resp = parse_review_plain(text).unwrap();
1785 assert_eq!(resp.decision, "approve");
1786 }
1787
1788 #[serial(cooldown_state)]
1791 #[test]
1792 fn infer_review_response_negation_do_not_approve_backtick() {
1793 let text = "There are unresolved issues — do not `approve` this PR until they are fixed.";
1794 assert!(
1795 parse_review_plain(text).is_err(),
1796 "negative context with backtick approve must not infer approval"
1797 );
1798 }
1799
1800 fn parse_via_agent_path(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
1805 let text = find_agent_result(agent, output)
1806 .map(|r| r.result_text)
1807 .filter(|t| !t.is_empty())
1808 .unwrap_or_else(|| output.to_string());
1809 if let Ok(r) = parse_review_response(&text) {
1810 return Ok(r);
1811 }
1812 infer_review_response_from_text(&text)
1813 .ok_or_else(|| anyhow::anyhow!("failed to parse review response from {agent} output"))
1814 }
1815
1816 #[serial(cooldown_state)]
1817 #[test]
1818 fn agent_output_claude_ndjson_review() {
1819 let ndjson = concat!(
1820 r#"{"type":"system","subtype":"init","session_id":"s1"}"#,
1821 "\n",
1822 r#"{"type":"assistant","message":{"content":[{"type":"text","text":"Reviewing..."}]}}"#,
1823 "\n",
1824 r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}","usage":{"input_tokens":100,"output_tokens":20}}"#,
1825 );
1826 let resp = parse_via_agent_path("claude", ndjson).unwrap();
1827 assert_eq!(resp.decision, "approve");
1828 }
1829
1830 #[serial(cooldown_state)]
1831 #[test]
1832 fn agent_output_opencode_ndjson_review() {
1833 let ndjson = concat!(
1834 r#"{"type":"step_start","timestamp":1000}"#,
1835 "\n",
1836 r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}"}}"#,
1837 "\n",
1838 r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop","tokens":{"input":100,"output":10}}}"#,
1839 );
1840 let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1841 assert_eq!(resp.decision, "request_changes");
1842 }
1843
1844 #[serial(cooldown_state)]
1845 #[test]
1846 fn agent_output_codex_ndjson_review() {
1847 let ndjson = concat!(
1848 r#"{"type":"thread.started","thread_id":"t1"}"#,
1849 "\n",
1850 r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All good\",\"issues\":[]}"}}"#,
1851 "\n",
1852 r#"{"type":"turn.completed"}"#,
1853 );
1854 let resp = parse_via_agent_path("codex", ndjson).unwrap();
1855 assert_eq!(resp.decision, "approve");
1856 }
1857
1858 #[serial(cooldown_state)]
1859 #[test]
1860 fn agent_output_plain_text_keyword_fallback() {
1861 let text = "All checks passed, LGTM.";
1862 let resp = parse_via_agent_path("claude", text).unwrap();
1863 assert_eq!(resp.decision, "approve");
1864 }
1865
1866 #[serial(cooldown_state)]
1867 #[test]
1868 fn agent_output_direct_json() {
1869 let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1870 let resp = parse_via_agent_path("opencode", json).unwrap();
1871 assert_eq!(resp.decision, "approve");
1872 }
1873
1874 #[serial(cooldown_state)]
1878 #[test]
1879 fn agent_output_text_with_json_fragment_no_false_positive() {
1880 let text = r#"I checked the file and found {"type":"text"} in the output. No issues."#;
1881 let result = parse_via_agent_path("claude", text);
1885 assert!(
1887 result.is_err() || result.as_ref().is_ok_and(|r| r.decision != "text"),
1888 "should not extract review from embedded JSON fragment"
1889 );
1890 }
1891
1892 #[serial(cooldown_state)]
1895 #[test]
1896 fn parse_review_response_ignores_prose_with_decision_word() {
1897 let text = r#"The decision was made. Here is a debug trace: {"trace":123}"#;
1898 assert!(
1899 parse_review_response(text).is_err(),
1900 "prose with 'decision' word and unrelated JSON should not parse as ReviewResponse"
1901 );
1902 }
1903
1904 #[serial(cooldown_state)]
1907 #[test]
1908 fn parse_review_response_ignores_embedded_json_without_decision_key() {
1909 let text = r#"Analysis complete. Config: {"timeout":30,"retries":3}. Done."#;
1910 assert!(
1911 parse_review_response(text).is_err(),
1912 "embedded JSON without 'decision' key should not parse as ReviewResponse"
1913 );
1914 }
1915
1916 #[serial(cooldown_state)]
1919 #[test]
1920 fn read_file_with_limit_reads_small_file_completely() {
1921 let temp_dir = env::temp_dir();
1922 let file_path = temp_dir.join(format!("orch_test_small_{}.txt", process::id()));
1923 let content = "Hello, World!";
1924 fs::write(&file_path, content).unwrap();
1925
1926 let result = read_file_with_limit(&file_path, 1024);
1927 assert!(result.is_ok());
1928 assert_eq!(result.unwrap(), content);
1929
1930 let _ = fs::remove_file(&file_path);
1931 }
1932
1933 #[serial(cooldown_state)]
1934 #[test]
1935 fn read_file_with_limit_truncates_large_file() {
1936 let temp_dir = env::temp_dir();
1937 let file_path = temp_dir.join(format!("orch_test_large_{}.txt", process::id()));
1938
1939 let large_content = "x".repeat(200);
1941 fs::write(&file_path, &large_content).unwrap();
1942
1943 let result = read_file_with_limit(&file_path, 50);
1945 assert!(result.is_ok());
1946
1947 let read_content = result.unwrap();
1948 assert_eq!(read_content.len(), 50);
1950 assert_eq!(read_content, "x".repeat(50));
1951
1952 let _ = fs::remove_file(&file_path);
1953 }
1954
1955 #[serial(cooldown_state)]
1956 #[test]
1957 fn read_file_with_limit_handles_empty_file() {
1958 let temp_dir = env::temp_dir();
1959 let file_path = temp_dir.join(format!("orch_test_empty_{}.txt", process::id()));
1960 fs::write(&file_path, "").unwrap();
1961
1962 let result = read_file_with_limit(&file_path, 1024);
1963 assert!(result.is_ok());
1964 assert_eq!(result.unwrap(), "");
1965
1966 let _ = fs::remove_file(&file_path);
1967 }
1968
1969 #[serial(cooldown_state)]
1970 #[test]
1971 fn read_file_with_limit_handles_nonexistent_file() {
1972 let temp_dir = env::temp_dir();
1973 let file_path = temp_dir.join("orch_test_nonexistent_xyz123.txt");
1974
1975 let result = read_file_with_limit(&file_path, 1024);
1976 assert!(result.is_err());
1977 }
1978
1979 #[serial(cooldown_state)]
1980 #[test]
1981 fn read_file_with_limit_reads_exact_limit() {
1982 let temp_dir = env::temp_dir();
1983 let file_path = temp_dir.join(format!("orch_test_exact_{}.txt", process::id()));
1984
1985 let content = "y".repeat(100);
1987 fs::write(&file_path, &content).unwrap();
1988
1989 let result = read_file_with_limit(&file_path, 100);
1991 assert!(result.is_ok());
1992 assert_eq!(result.unwrap().len(), 100);
1993
1994 let _ = fs::remove_file(&file_path);
1995 }
1996
1997 #[serial(cooldown_state)]
1998 #[test]
1999 fn read_file_with_limit_handles_unicode_truncation() {
2000 let temp_dir = env::temp_dir();
2001 let file_path = temp_dir.join(format!("orch_test_unicode_{}.txt", process::id()));
2002
2003 let emoji = "🎉".repeat(100); fs::write(&file_path, &emoji).unwrap();
2006
2007 let result = read_file_with_limit(&file_path, 50);
2009 assert!(result.is_ok());
2010
2011 let read_content = result.unwrap();
2012 assert_ne!(read_content, "");
2014
2015 let _ = fs::remove_file(&file_path);
2016 }
2017
2018 fn parse_review_via_agent(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
2022 let text = find_agent_result(agent, output)
2023 .map(|r| r.result_text)
2024 .filter(|t| !t.is_empty())
2025 .unwrap_or_else(|| output.to_string());
2026 if let Ok(r) = parse_review_response(&text) {
2027 return Ok(r);
2028 }
2029 infer_review_response(&text)
2030 .ok_or_else(|| anyhow::anyhow!("failed to parse review from {agent} output"))
2031 }
2032
2033 #[serial(cooldown_state)]
2036 #[test]
2037 fn fixture_claude_review_approve() {
2038 let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2039 let resp = parse_review_via_agent("claude", raw).unwrap();
2040 assert_eq!(resp.decision, "approve");
2041 assert_eq!(resp.notes, "All checks pass");
2042 assert_eq!(resp.test_results.as_deref(), Some("pass"));
2043 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2044 }
2045
2046 #[serial(cooldown_state)]
2047 #[test]
2048 fn fixture_claude_review_request_changes() {
2049 let raw = include_str!("../../../tests/fixtures/review_claude_request_changes.jsonl");
2050 let resp = parse_review_via_agent("claude", raw).unwrap();
2051 assert_eq!(resp.decision, "request_changes");
2052 assert_eq!(resp.notes, "Fix the null deref on line 42");
2053 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2054 assert_eq!(resp.issues.len(), 1);
2055 assert_eq!(resp.issues[0].file, "src/main.rs");
2056 assert_eq!(resp.issues[0].line, Some(42));
2057 assert_eq!(resp.issues[0].severity, "error");
2058 }
2059
2060 #[serial(cooldown_state)]
2061 #[test]
2062 fn fixture_claude_review_rate_limit_returns_error() {
2063 let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2066 let result = find_agent_result("claude", raw);
2067 assert!(
2068 result.is_some(),
2069 "should find result even for error envelope"
2070 );
2071 let result = result.unwrap();
2072 assert!(result.is_error, "rate limit should be flagged as error");
2073 assert!(result.result_text.contains("rate limit"));
2074 }
2075
2076 #[serial(cooldown_state)]
2079 #[test]
2080 fn fixture_opencode_review_approve() {
2081 let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2082 let resp = parse_review_via_agent("opencode", raw).unwrap();
2083 assert_eq!(resp.decision, "approve");
2084 assert_eq!(resp.notes, "All checks pass");
2085 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2086 }
2087
2088 #[serial(cooldown_state)]
2089 #[test]
2090 fn fixture_opencode_review_request_changes() {
2091 let raw = include_str!("../../../tests/fixtures/review_opencode_request_changes.jsonl");
2092 let resp = parse_review_via_agent("opencode", raw).unwrap();
2093 assert_eq!(resp.decision, "request_changes");
2094 assert_eq!(resp.notes, "Fix the memory leak");
2095 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2096 assert_eq!(resp.issues.len(), 1);
2097 assert_eq!(resp.issues[0].file, "src/lib.rs");
2098 assert_eq!(resp.issues[0].line, Some(100));
2099 }
2100
2101 #[serial(cooldown_state)]
2102 #[test]
2103 fn fixture_opencode_review_plain_text_approve() {
2104 let raw = include_str!("../../../tests/fixtures/review_opencode_plain_text.jsonl");
2106 let resp = parse_review_via_agent("opencode", raw).unwrap();
2107 assert_eq!(resp.decision, "approve");
2108 }
2109
2110 #[serial(cooldown_state)]
2113 #[test]
2114 fn fixture_codex_review_approve() {
2115 let raw = include_str!("../../../tests/fixtures/review_codex_approve.jsonl");
2116 let resp = parse_review_via_agent("codex", raw).unwrap();
2117 assert_eq!(resp.decision, "approve");
2118 assert_eq!(resp.notes, "Clean implementation");
2119 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2120 }
2121
2122 #[serial(cooldown_state)]
2123 #[test]
2124 fn fixture_codex_review_request_changes() {
2125 let raw = include_str!("../../../tests/fixtures/review_codex_request_changes.jsonl");
2126 let resp = parse_review_via_agent("codex", raw).unwrap();
2127 assert_eq!(resp.decision, "request_changes");
2128 assert_eq!(resp.notes, "Race condition on line 15");
2129 assert_eq!(resp.issues.len(), 1);
2130 assert_eq!(resp.issues[0].file, "src/server.rs");
2131 assert_eq!(resp.issues[0].line, Some(15));
2132 }
2133
2134 #[serial(cooldown_state)]
2135 #[test]
2136 fn fixture_codex_review_plain_text_approve() {
2137 let raw = include_str!("../../../tests/fixtures/review_codex_plain_text.jsonl");
2139 let resp = parse_review_via_agent("codex", raw).unwrap();
2140 assert_eq!(resp.decision, "approve");
2141 }
2142
2143 #[serial(cooldown_state)]
2146 #[test]
2147 fn fixture_kimi_review_approve() {
2148 let raw = include_str!("../../../tests/fixtures/review_kimi_approve.jsonl");
2149 let resp = parse_review_via_agent("kimi", raw).unwrap();
2150 assert_eq!(resp.decision, "approve");
2151 assert_eq!(resp.notes, "LGTM");
2152 assert_eq!(resp.test_results.as_deref(), Some("pass"));
2153 assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2154 }
2155
2156 #[serial(cooldown_state)]
2159 #[test]
2160 fn fixture_minimax_review_request_changes() {
2161 let raw = include_str!("../../../tests/fixtures/review_minimax_request_changes.jsonl");
2162 let resp = parse_review_via_agent("minimax", raw).unwrap();
2163 assert_eq!(resp.decision, "request_changes");
2164 assert_eq!(resp.notes, "Fix the bug");
2165 assert_eq!(resp.test_results.as_deref(), Some("fail"));
2166 assert_eq!(resp.issues.len(), 1);
2167 assert_eq!(resp.issues[0].file, "src/foo.rs");
2168 assert_eq!(resp.issues[0].line, Some(42));
2169 assert_eq!(resp.issues[0].severity, "error");
2170 }
2171
2172 #[serial(cooldown_state)]
2176 #[test]
2177 fn find_agent_result_extracts_tokens_claude() {
2178 let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2179 let result = find_agent_result("claude", raw).expect("should find result");
2180 assert_eq!(result.input_tokens, Some(200));
2181 assert_eq!(result.output_tokens, Some(30));
2182 assert!(result.cost_usd.is_none()); }
2184
2185 #[serial(cooldown_state)]
2186 #[test]
2187 fn find_agent_result_extracts_tokens_opencode() {
2188 let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2189 let result = find_agent_result("opencode", raw).expect("should find result");
2190 assert_eq!(result.input_tokens, Some(2900));
2191 assert_eq!(result.output_tokens, Some(100));
2192 assert_eq!(result.cost_usd, Some(0.002));
2193 }
2194
2195 #[serial(cooldown_state)]
2197 #[test]
2198 fn find_agent_result_returns_none_for_empty() {
2199 assert!(find_agent_result("claude", "").is_none());
2200 assert!(find_agent_result("opencode", "").is_none());
2201 assert!(find_agent_result("codex", "").is_none());
2202 assert!(find_agent_result("kimi", "").is_none());
2203 assert!(find_agent_result("minimax", "").is_none());
2204 }
2205
2206 #[serial(cooldown_state)]
2207 #[test]
2208 fn find_agent_result_returns_none_for_plain_text() {
2209 assert!(find_agent_result("claude", "just some plain text").is_none());
2210 assert!(find_agent_result("codex", "just some plain text").is_none());
2211 }
2214
2215 #[serial(cooldown_state)]
2219 #[test]
2220 fn find_agent_result_propagates_claude_rate_limit() {
2221 let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2222 let result = find_agent_result("claude", raw).expect("should find result");
2223 assert!(result.is_error, "should be flagged as error");
2224 assert!(result.result_text.contains("rate limit"));
2225 }
2226
2227 #[cfg(unix)]
2230 fn failover_env() -> tempfile::TempDir {
2231 let dir = tempfile::tempdir().unwrap();
2232 let bin = dir.path().join("bin");
2233 fs::create_dir(&bin).unwrap();
2234 for agent in ["claude", "codex"] {
2235 let path = bin.join(agent);
2236 fs::write(&path, "#!/bin/sh\n").unwrap();
2237 fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap();
2238 }
2239 let home = dir.path().join("home");
2240 fs::create_dir(&home).unwrap();
2241 fs::write(
2242 home.join("config.yml"),
2243 "agents: [claude, codex]\ndispatch:\n retry_base_delay_ms: 1\n retry_max_delay_ms: 1\nworkflow:\n max_timeout_retries: 3\n",
2244 )
2245 .unwrap();
2246 let mut path = bin.as_os_str().to_os_string();
2247 path.push(":");
2248 path.push(env::var_os("PATH").unwrap_or_default());
2249 unsafe { env::set_var("PATH", path) };
2251 unsafe { env::set_var("ORCH_HOME", &home) };
2253 cmd_cache::reset_for_test();
2254 dir
2255 }
2256
2257 async fn failover_task() -> (Arc<TaskStore>, String) {
2258 let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
2259 let id = store
2260 .create_internal("owner/repo", "t", "b", "test", "", None)
2261 .await
2262 .unwrap();
2263 (store, format!("internal:{id}"))
2264 }
2265
2266 #[cfg(unix)]
2268 #[serial(cooldown_state)]
2269 #[tokio::test]
2270 async fn timeout_with_every_other_agent_cooled_retries_as_new() {
2271 let _env = failover_env();
2272 cooldown::reset_global_state().await;
2273 set_agent_cooldown("codex", 600).await;
2274 let (store, task_id) = failover_task().await;
2275 let store = Some(store);
2276
2277 let outcome = handle_failover(
2278 &task_id,
2279 "claude",
2280 RetryableError::Timeout,
2281 "claude timed out after 1804s",
2282 false,
2283 &store,
2284 "owner/repo",
2285 )
2286 .await;
2287
2288 assert_eq!(outcome.status, "new");
2289 assert!(outcome.next_agent.is_none());
2290 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2291 .await
2292 .unwrap();
2293 assert!(task.limit_reroute_chain.is_empty(), "chain must be cleared");
2294 assert!(
2295 task.last_error.contains("will retry"),
2296 "{}",
2297 task.last_error
2298 );
2299 assert_eq!(task.timeout_retries, 1);
2300 }
2301
2302 #[cfg(unix)]
2304 #[serial(cooldown_state)]
2305 #[tokio::test]
2306 async fn timeout_with_a_free_agent_reroutes_to_it() {
2307 let _env = failover_env();
2308 cooldown::reset_global_state().await;
2309 let (store, task_id) = failover_task().await;
2310 let store = Some(store);
2311
2312 let outcome = handle_failover(
2313 &task_id,
2314 "claude",
2315 RetryableError::Timeout,
2316 "claude timed out after 1804s",
2317 false,
2318 &store,
2319 "owner/repo",
2320 )
2321 .await;
2322
2323 assert_eq!(outcome.status, "new");
2324 assert_eq!(outcome.next_agent.as_deref(), Some("codex"));
2325 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2326 .await
2327 .unwrap();
2328 assert_eq!(task.limit_reroute_chain, "claude");
2329 assert_eq!(task.agent.as_deref(), Some("codex"));
2330 assert!(
2331 task.model.as_deref().unwrap_or("").is_empty(),
2332 "model must be cleared so routing picks one for the fallback, got {:?}",
2333 task.model
2334 );
2335 assert!(
2336 task.last_error.contains("switching to codex"),
2337 "{}",
2338 task.last_error
2339 );
2340 }
2341
2342 #[cfg(unix)]
2344 #[serial(cooldown_state)]
2345 #[tokio::test]
2346 async fn timeout_failover_keeps_fallback_after_reroute_clear() {
2347 let _env = failover_env();
2348 cooldown::reset_global_state().await;
2349 let (store_arc, task_id) = failover_task().await;
2350 let store_id = store_arc
2351 .resolve_task_id("owner/repo", &task_id)
2352 .await
2353 .unwrap()
2354 .unwrap();
2355 store_arc
2356 .update_status(store_id, TaskStatus::InProgress)
2357 .await
2358 .unwrap();
2359 store_arc
2360 .set_fields(store_id, &[("agent", serde_json::json!("claude"))])
2361 .await
2362 .unwrap();
2363
2364 let store = Some(Arc::clone(&store_arc));
2365 let outcome = handle_failover(
2366 &task_id,
2367 "claude",
2368 RetryableError::Timeout,
2369 "claude timed out after 1804s",
2370 false,
2371 &store,
2372 "owner/repo",
2373 )
2374 .await;
2375 assert_eq!(outcome.status, "new");
2376 assert_eq!(outcome.next_agent.as_deref(), Some("codex"));
2377
2378 let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store_arc));
2379 runner
2380 .clear_agent_on_reroute_unless_fallback(&task_id, "claude")
2381 .await;
2382
2383 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2384 .await
2385 .unwrap();
2386 assert_eq!(task.agent.as_deref(), Some("codex"));
2387 assert_eq!(task.status, TaskStatus::InProgress);
2388 assert!(
2389 task.model.as_deref().unwrap_or("").is_empty(),
2390 "model must stay empty so routing picks one for the fallback, got {:?}",
2391 task.model
2392 );
2393
2394 let mut config = RouterConfig::default();
2395 let medium = config.model_map.entry("medium".to_string()).or_default();
2396 medium.insert("claude".to_string(), vec!["haiku".to_string()]);
2397 medium.insert("codex".to_string(), vec!["gpt-test".to_string()]);
2398 let agents = vec!["claude".to_string(), "codex".to_string()];
2399 let mut router = Router::new_for_test(config, agents);
2400 let labeled = ExternalTask {
2401 id: ExternalId(task_id.clone()),
2402 title: "Test".to_string(),
2403 body: "Test body".to_string(),
2404 state: "open".to_string(),
2405 labels: vec!["agent:claude".to_string()],
2406 author: "test".to_string(),
2407 created_at: "2026-01-01T00:00:00Z".to_string(),
2408 updated_at: "2026-01-01T00:00:00Z".to_string(),
2409 url: format!("https://github.com/owner/repo/issues/{task_id}"),
2410 };
2411 let result = router
2412 .route(&labeled, &store_arc, "owner/repo")
2413 .await
2414 .unwrap();
2415 assert_eq!(result.agent, "codex");
2416 }
2417
2418 #[cfg(unix)]
2420 #[serial(cooldown_state)]
2421 #[tokio::test]
2422 async fn repeated_timeouts_block_the_task() {
2423 let _env = failover_env();
2424 cooldown::reset_global_state().await;
2425 set_agent_cooldown("codex", 600).await;
2426 let (store, task_id) = failover_task().await;
2427 let store = Some(store);
2428
2429 let mut statuses = Vec::new();
2430 for _ in 0..3 {
2431 statuses.push(
2432 handle_failover(
2433 &task_id,
2434 "claude",
2435 RetryableError::Timeout,
2436 "claude timed out after 1804s",
2437 false,
2438 &store,
2439 "owner/repo",
2440 )
2441 .await
2442 .status,
2443 );
2444 }
2445
2446 assert_eq!(statuses, ["new", "new", "blocked"]);
2447 let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2448 .await
2449 .unwrap();
2450 assert_eq!(task.block_reason.as_deref(), Some("timed out 3 times"));
2451 assert_eq!(task.timeout_retries, 3);
2452 }
2453}