Skip to main content

orch/engine/runner/
response.rs

1//! Response collection + error classification.
2//!
3//! After the agent finishes (tmux session ends), this module:
4//! 1. Reads the output file
5//! 2. Parses the response JSON
6//! 3. Classifies errors (timeout, usage limit, auth, tooling)
7//! 4. Determines next action (success, reroute, needs_review)
8
9use crate::store;
10use crate::store::TaskStore;
11use std::collections::HashSet;
12use std::collections::hash_map::DefaultHasher;
13use std::fs;
14use std::fs::File;
15use std::hash::Hash;
16use std::hash::Hasher;
17use std::io::Read;
18use std::io::Seek;
19use std::io::SeekFrom;
20use std::path::{Path, PathBuf};
21use std::result;
22use std::sync::Arc;
23use std::thread;
24use std::time::Duration;
25use std::time::SystemTime;
26
27use tokio::fs as tokio_fs;
28use tokio::task;
29use tokio::time;
30// Cooldown tracking is implemented in `crate::engine::cooldown` (shared with the router).
31use crate::cli::ndjson;
32use crate::cmd_cache;
33use crate::config as app_config;
34use crate::engine;
35pub use crate::engine::cooldown::{
36    clear_expired_cooldowns, is_agent_in_cooldown, is_model_in_cooldown,
37    record_agent_failure_with_message, record_model_failure, set_agent_cooldown,
38};
39use crate::engine::router::config;
40use crate::home;
41use crate::parser::AgentResponse;
42use crate::store::MemoryEntry;
43use crate::store::Task;
44
45const JITTER_FACTOR: f64 = 0.3;
46
47/// Maximum output file size to read (10 MB).
48/// Prevents runaway outputs from consuming excessive memory.
49const MAX_OUTPUT_SIZE_BYTES: u64 = 10 * 1024 * 1024;
50
51/// Read a file with a size limit, returning truncated content if the file is too large.
52///
53/// This prevents runaway agent outputs from causing memory issues. If the file
54/// exceeds the limit, reads from the end of the file to capture the most recent
55/// output (which is typically where the result or error message is).
56fn read_file_with_limit<P: AsRef<Path>>(path: P, max_bytes: u64) -> anyhow::Result<String> {
57    let path = path.as_ref();
58    let metadata = fs::metadata(path)?;
59
60    if !metadata.is_file() {
61        anyhow::bail!("not a file: {}", path.display());
62    }
63
64    let file_size = metadata.len();
65
66    // If file is within limits, read the whole thing
67    if file_size <= max_bytes {
68        return fs::read_to_string(path).map_err(Into::into);
69    }
70
71    // File is too large - read from the end to get the most recent output
72    let mut file = File::open(path)?;
73    let start_pos = file_size.saturating_sub(max_bytes);
74    file.seek(SeekFrom::Start(start_pos))?;
75
76    let mut buffer = Vec::with_capacity(usize::try_from(max_bytes).unwrap_or(0));
77    file.read_to_end(&mut buffer)?;
78
79    // Try to convert to UTF-8, replacing invalid sequences
80    let content = String::from_utf8_lossy(&buffer);
81
82    // Log a warning about truncation
83    tracing::warn!(
84        path = %path.display(),
85        file_size = file_size,
86        max_bytes = max_bytes,
87        "output file truncated: exceeded size limit"
88    );
89
90    Ok(content.into_owned())
91}
92
93/// Calculate exponential backoff with jitter for fallback retries.
94/// Uses retry count from the reroute chain to determine the delay.
95///
96/// Delay = min(base * 2^retry_count, max) * (1 + jitter)
97pub(crate) fn calculate_backoff_delay(retry_count: usize) -> u64 {
98    let base_delay = config::retry_base_delay_ms();
99    let max_delay = config::retry_max_delay_ms();
100
101    // Exponential: base * 2^retry_count
102    let exponential =
103        base_delay * (2_u64.saturating_pow(u32::try_from(retry_count).unwrap_or(u32::MAX)));
104    let capped = exponential.min(max_delay);
105
106    // Add jitter: ±30% using a hash-based pseudo-random value.
107    // Mix system time (nanoseconds) with thread ID so concurrent retries on
108    // different threads get different delays even when called in the same
109    // microsecond, avoiding the thundering-herd problem.
110    #[expect(
111        clippy::cast_precision_loss,
112        clippy::cast_possible_truncation,
113        clippy::cast_sign_loss,
114        reason = "a jitter range in milliseconds, far below 2^52 and never negative"
115    )]
116    let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
117    let jitter = {
118        let mut h = DefaultHasher::new();
119        SystemTime::now().hash(&mut h);
120        thread::current().id().hash(&mut h);
121        h.finish() % (jitter_range * 2 + 1)
122    };
123
124    capped.saturating_sub(jitter_range) + jitter
125}
126
127fn fallback_retry_count(task: Option<&Task>) -> usize {
128    task.map_or(0, |t| {
129        let chain_retry_count = t
130            .limit_reroute_chain
131            .split(',')
132            .filter(|entry| !entry.is_empty())
133            .count();
134        let network_retry_count = usize::try_from(t.network_retries).unwrap_or(0);
135        chain_retry_count.max(network_retry_count)
136    })
137}
138
139/// Wait with exponential backoff before retrying with a fallback agent.
140/// Returns the delay applied in milliseconds.
141pub async fn wait_for_fallback_backoff(
142    task_id: &str,
143    store: &Option<Arc<TaskStore>>,
144    repo: &str,
145) -> u64 {
146    let task = store::opt_store_get_task(store, repo, task_id).await;
147
148    // Normal failover uses reroute chain length. NetworkError retries bypass
149    // failover, so track their own streak and use whichever retry source is larger.
150    let retry_count = fallback_retry_count(task.as_ref());
151
152    let delay = calculate_backoff_delay(retry_count);
153
154    if delay > 0 {
155        tracing::debug!(
156            task_id,
157            retry_count = retry_count,
158            delay_ms = delay,
159            "fallback backoff: waiting before retry"
160        );
161        time::sleep(Duration::from_millis(delay)).await;
162    }
163
164    delay
165}
166
167/// Outcome signal for the engine to update router weights.
168///
169/// Returned by the task runner so the engine can feed rate limit and
170/// success signals back to the router's weighted round-robin implementation.
171#[derive(Debug, Clone)]
172pub enum WeightSignal {
173    /// Agent completed a task successfully.
174    Success { agent: String },
175    /// Agent hit a rate limit / usage limit.
176    RateLimited { agent: String },
177    /// The run failed (timeout, no fallback, exhausted agents) and ended in `needs_review`.
178    /// Never read as a completed run, so the no-PR shortcut cannot mark the task done.
179    Failed { agent: String },
180    /// Task is blocked on child tasks (delegations). Do not trigger review agent.
181    Blocked,
182    /// Task was rerouted (silence detection, timeout, parse error) — reset to new for re-routing.
183    /// Distinct from None so the engine never converts a silent-reset task into needs_review.
184    Rerouted,
185    /// No weight-relevant signal (guard skipped task, unrecognised status, etc.)
186    None,
187}
188
189/// Read the agent's output file, trying multiple locations.
190///
191/// Checks per-task attempt directory first, then legacy flat paths.
192/// Output is truncated to `MAX_OUTPUT_SIZE_BYTES` to prevent runaway memory usage.
193pub async fn read_output_file(task_id: &str, primary_path: &Path, repo: &str) -> String {
194    // Primary: explicit output file (already points to attempt dir)
195    if let Some(content) = task::spawn_blocking({
196        let p = primary_path.to_path_buf();
197        move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
198    })
199    .await
200    .ok()
201    .and_then(result::Result::ok)
202        && !content.is_empty()
203    {
204        return content;
205    }
206
207    // Fallback: check all attempt dirs for this task (newest first)
208    if let Ok(task_dir) = home::task_dir_async(repo, task_id).await {
209        let attempts_dir = task_dir.join("attempts");
210        // Use async metadata for is_dir check
211        if tokio_fs::metadata(&attempts_dir)
212            .await
213            .is_ok_and(|m| m.is_dir())
214        {
215            let mut attempt_nums: Vec<u32> = task::spawn_blocking({
216                let a = attempts_dir.clone();
217                move || {
218                    match fs::read_dir(&a) {
219                        Ok(entries) => {
220                            let mut nums = Vec::new();
221                            for entry in entries {
222                                match entry {
223                                    Ok(e) => {
224                                        if let Some(n) =
225                                            e.file_name().to_str().and_then(|n| n.parse().ok())
226                                        {
227                                            nums.push(n);
228                                        }
229                                    }
230                                    Err(e) => tracing::warn!(
231                                        path = %a.display(),
232                                        err = %e,
233                                        "error reading attempt dir entry in output fallback"
234                                    ),
235                                }
236                            }
237                            nums
238                        }
239                        Err(e) => {
240                            tracing::warn!(path = %a.display(), err = %e, "failed to read attempts dir in output fallback");
241                            Vec::new()
242                        }
243                    }
244                }
245            })
246            .await
247            .unwrap_or_default();
248
249            attempt_nums.sort_unstable_by(|a, b| b.cmp(a)); // newest first
250            for n in attempt_nums {
251                let p = attempts_dir.join(n.to_string()).join("output.json");
252                if let Some(content) = task::spawn_blocking({
253                    let p = p.clone();
254                    move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
255                })
256                .await
257                .ok()
258                .and_then(result::Result::ok)
259                    && !content.is_empty()
260                {
261                    tracing::info!(task_id, path = %p.display(), "read output from attempt dir");
262                    return content;
263                }
264            }
265        }
266    }
267
268    // Legacy fallback locations
269    let state_dir = home::state_dir().unwrap_or_else(|_| PathBuf::from("/tmp"));
270
271    let mut fallbacks = vec![
272        PathBuf::from(format!("/tmp/output-{task_id}.json")),
273        state_dir.join(format!("output-{task_id}.json")),
274    ];
275
276    if let Ok(legacy_path) = home::state_file(&format!("output-{task_id}.json"))
277        && !fallbacks.contains(&legacy_path)
278    {
279        fallbacks.push(legacy_path);
280    }
281
282    for path in &fallbacks {
283        if let Some(content) = task::spawn_blocking({
284            let p = path.clone();
285            move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
286        })
287        .await
288        .ok()
289        .and_then(result::Result::ok)
290            && !content.is_empty()
291        {
292            tracing::info!(task_id, path = %path.display(), "read output from legacy fallback");
293            return content;
294        }
295    }
296
297    String::new()
298}
299
300/// Pick a fallback agent, avoiding agents already in the reroute chain and agents in cooldown.
301pub fn pick_fallback_agent(
302    current_agent: &str,
303    chain: &str,
304    available_agents: &[String],
305) -> Option<String> {
306    // Clear expired cooldowns first
307    clear_expired_cooldowns();
308
309    let chain_set: HashSet<&str> = if chain.is_empty() {
310        HashSet::new()
311    } else {
312        chain.split(',').collect()
313    };
314
315    // Prefer agents that are not in cooldown and are not heavily degraded
316    // from recent rate-limits. This prevents immediate failover to agents
317    // that have been heavily penalized and likely to fail as well.
318    for agent in available_agents {
319        if agent == current_agent || chain_set.contains(agent.as_str()) {
320            continue;
321        }
322        if is_agent_in_cooldown(agent) {
323            continue;
324        }
325        // Weight-based filtering was removed from this module because it used
326        // a default-constructed AgentWeights (dead code that always returned
327        // true). Keep failover behavior simple here: prefer the first agent
328        // that is not in cooldown and not in the reroute chain. If we later
329        // expose the router's AgentWeights snapshot to this module we can add
330        // weight-based avoidance back in a non-dead way.
331        return Some(agent.clone());
332    }
333
334    None
335}
336
337/// Retryable error types that should trigger agent failover.
338#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum RetryableError {
340    /// Agent timed out.
341    Timeout,
342    /// Run exited without a result envelope
343    Interrupted,
344    /// Usage/rate limit hit.
345    UsageLimit,
346    /// Auth/billing error.
347    AuthError,
348    /// General failure (non-retryable but we still try fallback agents).
349    Failed,
350    /// Missing tooling - fallback might help if another agent has the tool.
351    MissingTooling,
352    /// A specific model was unavailable (e.g. "Model not found"). The agent
353    /// itself is fine — only the model is in cooldown. Failover may still pick
354    /// a different agent, but we must not bump the agent-level failure counter.
355    ModelUnavailable,
356}
357
358impl RetryableError {
359    /// Return a short classified string for this error type.
360    /// Used in DB rate-limit metrics and test assertions.
361    pub fn type_str(self) -> &'static str {
362        match self {
363            RetryableError::Timeout => "timeout",
364            RetryableError::Interrupted => "interrupted",
365            RetryableError::UsageLimit => "rate_limit",
366            RetryableError::AuthError => "auth_error",
367            RetryableError::ModelUnavailable => "model_unavailable",
368            RetryableError::Failed | RetryableError::MissingTooling => "failed",
369        }
370    }
371}
372
373/**
374 * Handle failover for any retryable error type. Returns "new" on reroute (or
375 * retryable exhaustion), "blocked" on hard exhaustion, "needs_review" when no
376 * fallback agents are available. `credit_recorded` means the caller already ran
377 * `record_credit_exhaustion`; the generic `failure_count:` counter must not be
378 * bumped here or the next ordinary failure starts at the extended backoff tier
379 * (issue #3652). Rate-limit event DB recording is done by the caller (mod.rs)
380 **/
381pub async fn handle_failover(
382    task_id: &str,
383    agent_name: &str,
384    error_type: RetryableError,
385    error_message: &str,
386    credit_recorded: bool,
387    store: &Option<Arc<TaskStore>>,
388    repo: &str,
389) -> String {
390    // Get the reroute chain
391    let chain = get_reroute_chain(task_id, store, repo).await;
392    let chain = update_reroute_chain(task_id, agent_name, &chain, store, repo).await;
393
394    if error_type == RetryableError::Timeout
395        && let Some(status) =
396            block_after_repeated_timeouts(task_id, error_message, store, repo).await
397    {
398        return status;
399    }
400
401    // Get all available agents
402    let available: Vec<String> = engine::configured_agents()
403        .into_iter()
404        .filter(|a| cmd_cache::command_exists(a))
405        .collect();
406
407    // Check if we've exhausted all agents (build chain_set once)
408    let chain_set: HashSet<&str> = if chain.is_empty() {
409        HashSet::new()
410    } else {
411        chain.split(',').collect()
412    };
413    let is_exhausted = available.iter().all(|a| chain_set.contains(a.as_str()));
414
415    if is_exhausted {
416        // Retryable failures (rate limits, quota, model unavailable) will resolve once
417        // cooldowns expire — reset to "new" so the router picks it up again then.
418        // Hard failures (generic agent crash, missing tooling) won't improve on retry;
419        // block immediately with a clear reason rather than wasting 63 minutes in the
420        // review-refire loop before getting a misleading "review agent rebroadcast" message.
421        let is_retryable = matches!(
422            error_type,
423            RetryableError::UsageLimit
424                | RetryableError::AuthError
425                | RetryableError::ModelUnavailable
426                | RetryableError::Timeout
427                | RetryableError::Interrupted
428        );
429        let next_status = if is_retryable { "new" } else { "blocked" };
430
431        tracing::warn!(
432            task_id,
433            agent = agent_name,
434            error_type = ?error_type,
435            next_status,
436            "all agents exhausted"
437        );
438
439        let mut store_fields: Vec<(&str, serde_json::Value)> = Vec::new();
440
441        if is_retryable {
442            // Clear reroute chain so the next dispatch builds a fresh one rather than
443            // immediately seeing "all agents in chain" and exhausting again.
444            let msg = format!("{error_message} (all agents exhausted, will retry when available)");
445            store_fields.push(("last_error", serde_json::json!(msg)));
446            store_fields.push(("limit_reroute_chain", serde_json::json!("")));
447        } else {
448            let msg = format!("{error_message} (all agents exhausted)");
449            let block_reason = format!(
450                "all agents exhausted: {} — {}",
451                error_type.type_str(),
452                error_message
453            );
454            store_fields.push(("last_error", serde_json::json!(msg)));
455            store_fields.push(("block_reason", serde_json::json!(block_reason)));
456        }
457
458        if let Err(e) = store::store_set_result(store, repo, task_id, &store_fields).await {
459            tracing::warn!(task_id, err = %e, "failed to write agents_exhausted fields to store");
460        }
461
462        // Log reroute activity for exhausted agents
463        let details = serde_json::json!({
464            "failure_reason": error_type.type_str(),
465            "error_message": error_message,
466            "chain": chain,
467            "agents_exhausted": true,
468            "is_retryable": is_retryable,
469        });
470        store::store_log_activity(
471            store,
472            repo,
473            task_id,
474            "rerouted",
475            Some("in_progress"),
476            Some(next_status),
477            None::<&str>,
478            None::<&str>,
479            Some(&details),
480        )
481        .await;
482
483        return next_status.to_string();
484    }
485
486    // Pick a fallback agent
487    if let Some(next) = pick_fallback_agent(agent_name, &chain, &available) {
488        tracing::info!(
489            task_id,
490            from = agent_name,
491            to = next,
492            error_type = ?error_type,
493            "failover: switching to fallback agent"
494        );
495
496        // Timeouts and interrupts are not agent faults, skip the agent cooldown
497        if !matches!(
498            error_type,
499            RetryableError::MissingTooling
500                | RetryableError::ModelUnavailable
501                | RetryableError::Timeout
502                | RetryableError::Interrupted
503        ) {
504            // Credit errors skip the generic counter (already counted via
505            // credit_failure_count); the 120s cooldown below still applies
506            if !credit_recorded {
507                record_agent_failure_with_message(agent_name, error_message).await;
508            }
509
510            // Apply brief 120s cooldown on the failed agent so the router skips it
511            // on the next routing attempt. This prevents the router from immediately
512            // re-selecting the same agent after clearing both agent and model (#1492).
513            // Also skipped for ModelUnavailable: the agent is healthy.
514            set_agent_cooldown(agent_name, 120).await;
515        }
516
517        let msg = format!("{error_message}, clearing agent/model for re-route");
518        if let Err(e) = store::store_set_result(
519            store,
520            repo,
521            task_id,
522            &[
523                ("agent", serde_json::json!("")),
524                ("model", serde_json::json!("")),
525                ("last_error", serde_json::json!(msg)),
526            ],
527        )
528        .await
529        {
530            tracing::warn!(task_id, err = %e, "failed to write agent/model clear to store");
531        }
532
533        // Apply exponential backoff with jitter before retry.
534        // Return "new" (not "routed") so the task goes through Phase 3a routing
535        // again. This ensures the router actually re-selects a compatible agent+model
536        // via model_for_complexity() instead of bypassing routing and defaulting to
537        // claude in the dispatch loop (#1604).
538        wait_for_fallback_backoff(task_id, store, repo).await;
539
540        // Log reroute activity for successful failover
541        let details = serde_json::json!({
542            "failure_reason": error_type.type_str(),
543            "error_message": error_message,
544            "from_agent": agent_name,
545            "to_agent": next,
546            "chain": chain,
547        });
548        store::store_log_activity(
549            store,
550            repo,
551            task_id,
552            "rerouted",
553            Some("in_progress"),
554            Some("new"),
555            Some(&next),
556            None::<&str>,
557            Some(&details),
558        )
559        .await;
560
561        return "new".to_string();
562    }
563
564    // No fallback available
565    tracing::warn!(task_id, agent = agent_name, "no fallback agents available");
566
567    let is_retryable = matches!(
568        error_type,
569        RetryableError::UsageLimit
570            | RetryableError::AuthError
571            | RetryableError::ModelUnavailable
572            | RetryableError::Timeout
573            | RetryableError::Interrupted
574    );
575
576    // Timeouts and interrupts are not agent faults, skip the agent cooldown
577    if !matches!(
578        error_type,
579        RetryableError::MissingTooling
580            | RetryableError::ModelUnavailable
581            | RetryableError::Timeout
582            | RetryableError::Interrupted
583    ) && !credit_recorded
584    {
585        record_agent_failure_with_message(agent_name, error_message).await;
586    }
587
588    if is_retryable {
589        // The other agents are in cooldown, so wait out the backoff and let the router
590        // pick the task up again. A fresh chain keeps the next run from seeing every agent as tried
591        wait_for_fallback_backoff(task_id, store, repo).await;
592
593        let msg = format!("{error_message} (no fallback agents, will retry)");
594        if let Err(e) = store::store_set_result(
595            store,
596            repo,
597            task_id,
598            &[
599                ("last_error", serde_json::json!(msg)),
600                ("limit_reroute_chain", serde_json::json!("")),
601            ],
602        )
603        .await
604        {
605            tracing::warn!(task_id, err = %e, "failed to write no_fallback retry fields to store");
606        }
607
608        let details = serde_json::json!({
609            "failure_reason": error_type.type_str(),
610            "error_message": error_message,
611            "chain": chain,
612            "no_fallback_available": true,
613            "is_retryable": true,
614        });
615        store::store_log_activity(
616            store,
617            repo,
618            task_id,
619            "rerouted",
620            Some("in_progress"),
621            Some("new"),
622            None::<&str>,
623            None::<&str>,
624            Some(&details),
625        )
626        .await;
627
628        return "new".to_string();
629    }
630
631    let msg = format!("{error_message}, no fallback agents");
632    if let Err(e) = store::store_set_result(
633        store,
634        repo,
635        task_id,
636        &[("last_error", serde_json::json!(msg))],
637    )
638    .await
639    {
640        tracing::warn!(task_id, err = %e, "failed to write no_fallback last_error to store");
641    }
642
643    // Log reroute activity for no fallback available
644    let details = serde_json::json!({
645        "failure_reason": error_type.type_str(),
646        "error_message": error_message,
647        "chain": chain,
648        "no_fallback_available": true,
649    });
650    store::store_log_activity(
651        store,
652        repo,
653        task_id,
654        "rerouted",
655        Some("in_progress"),
656        Some("needs_review"),
657        None::<&str>,
658        None::<&str>,
659        Some(&details),
660    )
661    .await;
662
663    "needs_review".to_string()
664}
665
666/// Count a timeout against the task. Returns "blocked" once the task has timed out
667/// `workflow.max_timeout_retries` times (default 3), so a task that always times out does not loop
668pub async fn block_after_repeated_timeouts(
669    task_id: &str,
670    error_message: &str,
671    store: &Option<Arc<TaskStore>>,
672    repo: &str,
673) -> Option<String> {
674    let max: i64 = app_config::get("workflow.max_timeout_retries")
675        .ok()
676        .and_then(|v| v.parse().ok())
677        .unwrap_or(3);
678    let count = match store::store_increment(store, repo, task_id, "timeout_retries").await {
679        Ok(count) => i64::try_from(count).unwrap_or(i64::MAX),
680        Err(e) => {
681            tracing::warn!(task_id, err = %e, "failed to increment timeout_retries");
682            return None;
683        }
684    };
685    if count < max {
686        return None;
687    }
688
689    let block_reason = format!("timed out {count} times");
690    tracing::warn!(
691        task_id,
692        count,
693        max,
694        "timeout retries exhausted, blocking task"
695    );
696    if let Err(e) = store::store_set_result(
697        store,
698        repo,
699        task_id,
700        &[
701            (
702                "last_error",
703                serde_json::json!(format!("{error_message} ({block_reason})")),
704            ),
705            ("block_reason", serde_json::json!(block_reason)),
706            ("limit_reroute_chain", serde_json::json!("")),
707        ],
708    )
709    .await
710    {
711        tracing::warn!(task_id, err = %e, "failed to write timeout block fields to store");
712    }
713    let details = serde_json::json!({
714        "failure_reason": "timeout",
715        "error_message": error_message,
716        "timeout_retries": count,
717    });
718    store::store_log_activity(
719        store,
720        repo,
721        task_id,
722        "rerouted",
723        Some("in_progress"),
724        Some("blocked"),
725        None::<&str>,
726        None::<&str>,
727        Some(&details),
728    )
729    .await;
730    Some("blocked".to_string())
731}
732
733/// Get the reroute chain from store.
734pub async fn get_reroute_chain(
735    task_id: &str,
736    store: &Option<Arc<TaskStore>>,
737    repo: &str,
738) -> String {
739    store::opt_store_get_task(store, repo, task_id)
740        .await
741        .map(|t| t.limit_reroute_chain)
742        .unwrap_or_default()
743        .trim()
744        .to_string()
745}
746
747/// Update the reroute chain in store.
748pub async fn update_reroute_chain(
749    task_id: &str,
750    current_agent: &str,
751    existing_chain: &str,
752    store: &Option<Arc<TaskStore>>,
753    repo: &str,
754) -> String {
755    let mut chain = existing_chain.to_string();
756    if chain.is_empty() {
757        chain = current_agent.to_string();
758    } else if !chain.split(',').any(|a| a == current_agent) {
759        chain = format!("{chain},{current_agent}");
760    }
761
762    if let Err(e) = store::store_set_result(
763        store,
764        repo,
765        task_id,
766        &[("limit_reroute_chain", serde_json::json!(chain))],
767    )
768    .await
769    {
770        tracing::warn!(task_id, err = %e, "failed to write limit_reroute_chain to store");
771    }
772    chain
773}
774
775/// Review response from the review agent.
776#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
777pub struct ReviewResponse {
778    /// Decision: "approve" | "request_changes"
779    pub decision: String,
780    /// Detailed review feedback.
781    pub notes: String,
782    /// Test results: "pass" | "fail" | "skipped"
783    pub test_results: Option<String>,
784    /// List of issues found.
785    pub issues: Vec<ReviewIssue>,
786}
787
788/// A single issue found during review.
789#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
790pub struct ReviewIssue {
791    /// File path.
792    pub file: String,
793    /// Line number (optional).
794    pub line: Option<u32>,
795    /// Severity: "error" | "warning"
796    pub severity: String,
797    /// Description of the issue.
798    pub description: String,
799}
800
801/// Parse a review response from already-unwrapped text.
802///
803/// Expects the raw result text (already extracted from the agent envelope
804/// by the agent-specific parser). Tries direct JSON parse, then markdown
805/// code block extraction.
806pub fn parse_review_response(text: &str) -> anyhow::Result<ReviewResponse> {
807    let resp = parse_review_json(text)?;
808    if matches!(resp.decision.as_str(), "approve" | "request_changes") {
809        return Ok(resp);
810    }
811
812    // Weak models echo the prompt template ("approve|request_changes"), strip it so it
813    // does not match the "request_changes" keyword
814    let stripped = text.replace("approve|request_changes", "");
815    infer_review_response_from_text(&stripped)
816        .ok_or_else(|| anyhow::anyhow!("invalid review decision: {}", resp.decision))
817}
818
819fn parse_review_json(text: &str) -> anyhow::Result<ReviewResponse> {
820    // Try direct JSON parse
821    if let Ok(resp) = serde_json::from_str::<ReviewResponse>(text) {
822        return Ok(resp);
823    }
824
825    // Try extracting JSON from markdown code blocks
826    if let Some(json_str) = extract_json_block(text)
827        && let Ok(resp) = serde_json::from_str::<ReviewResponse>(&json_str)
828    {
829        return Ok(resp);
830    }
831
832    if let Some(resp) = extract_review_response_object(text) {
833        return Ok(resp);
834    }
835
836    anyhow::bail!("failed to parse review response")
837}
838
839fn infer_review_response_from_text(text: &str) -> Option<ReviewResponse> {
840    let lower = text.to_ascii_lowercase();
841    let has_changes_requested =
842        lower.contains("changes requested") || lower.contains("request_changes");
843    if has_changes_requested {
844        return Some(ReviewResponse {
845            decision: "request_changes".to_string(),
846            notes: "Inferred decision from plain-text review output.".to_string(),
847            test_results: None,
848            issues: Vec::new(),
849        });
850    }
851
852    // Approval only from an explicit decision token. "approved" and CI-evidence
853    // phrases ("all checks passed") are not a decision, a negation list cannot cover them
854    let positive_approval = lower
855        .split(|c: char| !c.is_ascii_alphabetic())
856        .any(|word| word == "lgtm")
857        || lower.contains("decision is `approve`")
858        || lower.contains("decision: `approve`")
859        || lower.contains("decision is approve")
860        || lower.contains("decision: approve");
861    if positive_approval {
862        return Some(ReviewResponse {
863            decision: "approve".to_string(),
864            notes: "Inferred decision from plain-text review output.".to_string(),
865            test_results: None,
866            issues: Vec::new(),
867        });
868    }
869
870    None
871}
872
873/// Public entry point for keyword-based review response inference.
874///
875/// Used by `review.rs` when `parse_review_response` fails on the extracted text.
876pub fn infer_review_response(text: &str) -> Option<ReviewResponse> {
877    infer_review_response_from_text(text)
878}
879
880/// Extract the first valid JSON object from markdown code blocks.
881///
882/// Searches for ```json blocks and returns the first one whose content
883/// starts with `{`, skipping non-JSON code blocks.
884fn extract_json_block(text: &str) -> Option<String> {
885    let mut search_from = 0;
886    while let Some(start) = text[search_from..].find("```json") {
887        let abs_start = search_from + start;
888        let after_tag = abs_start + "```json".len();
889        let Some(rest) = text[after_tag..].strip_prefix('\n') else {
890            search_from = after_tag;
891            continue;
892        };
893        let content_start = text.len() - rest.len();
894        let Some(end_offset) = rest.find("```") else {
895            search_from = content_start;
896            continue;
897        };
898        let end = content_start + end_offset;
899        let content = text[content_start..end].trim();
900        if content.starts_with('{') {
901            return Some(content.to_string());
902        }
903        search_from = end + 3;
904    }
905    None
906}
907
908/// Extract a ReviewResponse JSON object from arbitrary text.
909///
910/// Scans for balanced JSON objects and tries to parse them as ReviewResponse.
911fn extract_review_response_object(text: &str) -> Option<ReviewResponse> {
912    let mut start: Option<usize> = None;
913    let mut depth: i32 = 0;
914    let mut in_string = false;
915    let mut escape = false;
916
917    for (idx, ch) in text.char_indices() {
918        if start.is_some() {
919            if in_string {
920                if escape {
921                    escape = false;
922                } else if ch == '\\' {
923                    escape = true;
924                } else if ch == '"' {
925                    in_string = false;
926                }
927                continue;
928            }
929
930            match ch {
931                '"' => in_string = true,
932                '{' => depth += 1,
933                '}' => {
934                    depth -= 1;
935                    if depth == 0 {
936                        debug_assert!(
937                            start.is_some(),
938                            "start must be Some when depth returns to 0"
939                        );
940                        let start_idx = start.unwrap_or(0);
941                        let candidate = &text[start_idx..=idx];
942                        if let Ok(resp) = serde_json::from_str::<ReviewResponse>(candidate) {
943                            return Some(resp);
944                        }
945                        start = None;
946                    }
947                }
948                _ => {}
949            }
950            continue;
951        }
952
953        if ch == '{' {
954            start = Some(idx);
955            depth = 1;
956            in_string = false;
957            escape = false;
958        }
959    }
960
961    None
962}
963
964/// Extract learnings and store as memory for future attempts.
965#[expect(
966    clippy::too_many_arguments,
967    reason = "a memory entry records the task, attempt, agent, model and outcome"
968)]
969pub async fn store_learnings_from_response(
970    task_id: &str,
971    attempt: u32,
972    agent: &str,
973    model: Option<&str>,
974    response: &AgentResponse,
975    error: Option<&str>,
976    store: &Option<Arc<TaskStore>>,
977    repo: &str,
978    raw_stdout: &str,
979) {
980    // Use agent self-reported files; fall back to NDJSON tool-call extraction when empty.
981    let files_modified = if response.files.is_empty() {
982        let extracted = ndjson::extract_files_changed(raw_stdout);
983        if !extracted.is_empty() {
984            tracing::debug!(
985                task_id,
986                count = extracted.len(),
987                "files_modified: fell back to NDJSON extraction (agent reported none)"
988            );
989        }
990        extracted
991    } else {
992        response.files.clone()
993    };
994
995    // Build the memory entry
996    let entry = MemoryEntry {
997        attempt,
998        agent: agent.to_string(),
999        model: model.map(String::from),
1000        learnings: response.learnings.clone(),
1001        error: error.map(String::from),
1002        files_modified,
1003        approach: response.summary.clone(),
1004        timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1005    };
1006
1007    if let Some(st) = store
1008        && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1009    {
1010        match st.append_memory(store_id, &entry).await {
1011            Err(e) => {
1012                tracing::warn!(task_id, error = ?e, "failed to store memory");
1013            }
1014            _ => {
1015                tracing::debug!(task_id, attempt, "stored memory for attempt");
1016            }
1017        }
1018    }
1019}
1020
1021/// Store a memory entry for a failed attempt (without a full AgentResponse).
1022pub async fn store_failure_memory(
1023    task_id: &str,
1024    attempt: u32,
1025    agent: &str,
1026    model: Option<&str>,
1027    error: &str,
1028    store: &Option<Arc<TaskStore>>,
1029    repo: &str,
1030) {
1031    let entry = MemoryEntry {
1032        attempt,
1033        agent: agent.to_string(),
1034        model: model.map(String::from),
1035        learnings: vec![],
1036        error: Some(error.to_string()),
1037        files_modified: vec![],
1038        approach: String::new(),
1039        timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1040    };
1041
1042    if let Some(st) = store
1043        && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1044    {
1045        match st.append_memory(store_id, &entry).await {
1046            Err(e) => {
1047                tracing::warn!(task_id, error = ?e, "failed to store failure memory");
1048            }
1049            _ => {
1050                tracing::debug!(task_id, attempt, "stored failure memory");
1051            }
1052        }
1053    }
1054}
1055
1056#[cfg(test)]
1057mod tests {
1058    use std::env;
1059    #[cfg(unix)]
1060    use std::os::unix::fs::PermissionsExt;
1061    use std::process;
1062
1063    use super::*;
1064    use crate::engine::cooldown;
1065    use crate::engine::runner::agents::{AgentError, find_agent_result, patterns};
1066    use crate::store::TaskStatus;
1067    use serial_test::serial;
1068
1069    #[serial(cooldown_state)]
1070    #[test]
1071    fn calculate_backoff_delay_jitter_is_centered() {
1072        // Run many samples and verify the range is [0.7*capped, 1.3*capped].
1073        // We can't control `now` directly, but calling the function many times
1074        // exercises different microsecond values and lets us check bounds.
1075        let base = config::retry_base_delay_ms();
1076        let max = config::retry_max_delay_ms();
1077
1078        // retry_count=0 → exponential = base, capped = base (assuming base < max)
1079        let capped = base.min(max);
1080        #[expect(
1081            clippy::cast_precision_loss,
1082            clippy::cast_possible_truncation,
1083            clippy::cast_sign_loss,
1084            reason = "mirrors the jitter range in calculate_backoff_delay"
1085        )]
1086        let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
1087        let lo = capped.saturating_sub(jitter_range);
1088        let hi = capped + jitter_range;
1089
1090        for _ in 0..200 {
1091            let delay = calculate_backoff_delay(0);
1092            assert!(
1093                delay >= lo && delay <= hi,
1094                "delay {delay} out of expected range [{lo}, {hi}]"
1095            );
1096        }
1097    }
1098
1099    #[serial(cooldown_state)]
1100    #[test]
1101    fn fallback_retry_count_uses_larger_network_retry_streak() {
1102        let task = Task {
1103            id: 1,
1104            external_id: None,
1105            repo: "owner/repo".to_string(),
1106            origin: "internal".to_string(),
1107            title: "test".to_string(),
1108            body: String::new(),
1109            status: TaskStatus::New,
1110            source: String::new(),
1111            source_id: String::new(),
1112            author: String::new(),
1113            url: String::new(),
1114            labels: vec![],
1115            agent: None,
1116            model: None,
1117            complexity: "simple".to_string(),
1118            estimate: 0,
1119            route_reason: String::new(),
1120            agent_profile: String::new(),
1121            selected_skills: String::new(),
1122            route_attempts: 0,
1123            attempts: 0,
1124            branch: String::new(),
1125            worktree: String::new(),
1126            worktree_cleaned: false,
1127            owner_node: None,
1128            summary: String::new(),
1129            last_error: String::new(),
1130            parent_id: None,
1131            block_reason: None,
1132            pr_number: None,
1133            pr_review_context: String::new(),
1134            last_review_ts: String::new(),
1135            review_ts_map: "{}".to_string(),
1136            last_comment_review_ts: String::new(),
1137            merge_conflict_retries: 0,
1138            ci_merge_failures: 0,
1139            pr_create_failures: 0,
1140            push_failures: 0,
1141            network_retries: 3,
1142            timeout_retries: 0,
1143            review_agent_failures: 0,
1144            review_resets: 0,
1145            review_cycles: 0,
1146            review_invocations: 0,
1147            needs_review_refires: 0,
1148            input_tokens: 0,
1149            output_tokens: 0,
1150            total_cost_usd: 0.0,
1151            model_reroute_chain: String::new(),
1152            limit_reroute_chain: "claude".to_string(),
1153            memory: vec![],
1154            delegations: vec![],
1155            auto_unblock_count: 0,
1156            auto_unblock_last_at: String::new(),
1157            auto_unblock_last_reason: String::new(),
1158            ci_recovery_count: 0,
1159            no_code_reroutes: 0,
1160            no_code_last_agent: String::new(),
1161            created_at: String::new(),
1162            updated_at: String::new(),
1163        };
1164
1165        assert_eq!(fallback_retry_count(Some(&task)), 3);
1166    }
1167
1168    #[serial(cooldown_state)]
1169    #[test]
1170    fn retryable_type_str_returns_correct_labels() {
1171        assert_eq!(RetryableError::Timeout.type_str(), "timeout");
1172        assert_eq!(RetryableError::Interrupted.type_str(), "interrupted");
1173        assert_eq!(RetryableError::UsageLimit.type_str(), "rate_limit");
1174        assert_eq!(RetryableError::AuthError.type_str(), "auth_error");
1175        assert_eq!(RetryableError::Failed.type_str(), "failed");
1176        assert_eq!(RetryableError::MissingTooling.type_str(), "failed");
1177    }
1178
1179    #[serial(cooldown_state)]
1180    #[test]
1181    fn is_usage_limit_detects_rate_limit() {
1182        assert!(patterns::detect_rate_limit("Error: rate limit exceeded").is_some());
1183        assert!(patterns::detect_rate_limit("HTTP 429 Too Many Requests").is_some());
1184        assert!(patterns::detect_rate_limit("quota exceeded for model").is_some());
1185        // context_length_exceeded is handled by detect_context_overflow, not detect_rate_limit
1186        assert!(patterns::detect_context_overflow("context_length_exceeded").is_some());
1187    }
1188
1189    #[serial(cooldown_state)]
1190    #[test]
1191    fn is_usage_limit_rejects_normal_text() {
1192        assert!(patterns::detect_rate_limit("task completed successfully").is_none());
1193        assert!(patterns::detect_rate_limit("").is_none());
1194    }
1195
1196    #[serial(cooldown_state)]
1197    #[test]
1198    fn is_auth_error_detects_common_patterns() {
1199        assert!(patterns::detect_auth_error("401 Unauthorized").is_some());
1200        assert!(patterns::detect_auth_error("invalid api key provided").is_some());
1201        assert!(patterns::detect_auth_error("Your billing plan has expired").is_some());
1202        assert!(patterns::detect_auth_error("Error: 403 Forbidden").is_some());
1203    }
1204
1205    #[serial(cooldown_state)]
1206    #[test]
1207    fn is_auth_error_rejects_normal_text() {
1208        assert!(patterns::detect_auth_error("task completed successfully").is_none());
1209        assert!(patterns::detect_auth_error("").is_none());
1210    }
1211
1212    #[serial(cooldown_state)]
1213    #[test]
1214    fn detect_missing_tooling_finds_known_tools() {
1215        let result = patterns::detect_missing_tool("bun: command not found");
1216        assert!(result.is_some());
1217        if let AgentError::MissingTool { tool } = result.unwrap() {
1218            assert_eq!(tool, "bun");
1219        } else {
1220            panic!("expected MissingTool error");
1221        }
1222
1223        let result = patterns::detect_missing_tool("env: anchor: no such file");
1224        assert!(result.is_some());
1225        if let AgentError::MissingTool { tool } = result.unwrap() {
1226            assert_eq!(tool, "anchor");
1227        } else {
1228            panic!("expected MissingTool error");
1229        }
1230
1231        let result = patterns::detect_missing_tool("spawn docker enoent");
1232        assert!(result.is_some());
1233        if let AgentError::MissingTool { tool } = result.unwrap() {
1234            assert_eq!(tool, "docker");
1235        } else {
1236            panic!("expected MissingTool error");
1237        }
1238    }
1239
1240    #[serial(cooldown_state)]
1241    #[test]
1242    fn detect_missing_tooling_returns_none_for_normal() {
1243        assert!(patterns::detect_missing_tool("everything works fine").is_none());
1244        assert!(patterns::detect_missing_tool("").is_none());
1245    }
1246
1247    // ── Cooldown tests ────────────────────────────────────────────
1248
1249    #[serial(cooldown_state)]
1250    #[tokio::test]
1251    async fn record_and_check_agent_cooldown() {
1252        cooldown::reset_global_state().await;
1253        // Use unique names to avoid interference from other tests
1254        let agent = "test_cooldown_agent_1";
1255        assert!(!is_agent_in_cooldown(agent));
1256        record_agent_failure_with_message(agent, "").await;
1257        assert!(is_agent_in_cooldown(agent));
1258    }
1259
1260    #[serial(cooldown_state)]
1261    #[tokio::test]
1262    async fn record_and_check_model_cooldown() {
1263        cooldown::reset_global_state().await;
1264        let agent = "test_cooldown_agent_2";
1265        let model = "test_model_x";
1266        assert!(!is_model_in_cooldown(agent, model));
1267        record_model_failure(agent, model).await;
1268        assert!(is_model_in_cooldown(agent, model));
1269        // Different model should not be in cooldown
1270        assert!(!is_model_in_cooldown(agent, "other_model"));
1271    }
1272
1273    #[serial(cooldown_state)]
1274    #[tokio::test]
1275    async fn clear_expired_does_not_remove_fresh_entries() {
1276        cooldown::reset_global_state().await;
1277        let agent = "test_cooldown_agent_3";
1278        record_agent_failure_with_message(agent, "").await;
1279        clear_expired_cooldowns();
1280        // Should still be in cooldown (just recorded)
1281        assert!(is_agent_in_cooldown(agent));
1282    }
1283
1284    // Use fake agent names so other tests don't interfere.
1285    #[serial(cooldown_state)]
1286    #[test]
1287    fn pick_fallback_skips_current_agent() {
1288        let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1289        let result = pick_fallback_agent("test_agent_a", "", &available);
1290        assert_eq!(result, Some("test_agent_b".to_string()));
1291    }
1292
1293    #[serial(cooldown_state)]
1294    #[test]
1295    fn pick_fallback_skips_chain_agents() {
1296        let available = vec![
1297            "test_agent_a".to_string(),
1298            "test_agent_b".to_string(),
1299            "test_agent_c".to_string(),
1300        ];
1301        let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1302        assert_eq!(result, Some("test_agent_c".to_string()));
1303    }
1304
1305    #[serial(cooldown_state)]
1306    #[test]
1307    fn pick_fallback_returns_none_when_exhausted() {
1308        let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1309        let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1310        assert!(result.is_none());
1311    }
1312
1313    #[serial(cooldown_state)]
1314    #[test]
1315    fn weight_signal_variants() {
1316        let success = WeightSignal::Success {
1317            agent: "claude".to_string(),
1318        };
1319        let limited = WeightSignal::RateLimited {
1320            agent: "codex".to_string(),
1321        };
1322        let none = WeightSignal::None;
1323
1324        // Verify Debug trait
1325        assert!(format!("{success:?}").contains("claude"));
1326        assert!(format!("{limited:?}").contains("codex"));
1327        assert!(format!("{none:?}").contains("None"));
1328    }
1329
1330    #[serial(cooldown_state)]
1331    #[test]
1332    fn parse_review_response_direct_json() {
1333        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1334        let resp = parse_review_response(json).unwrap();
1335        assert_eq!(resp.decision, "approve");
1336        assert_eq!(resp.notes, "LGTM");
1337        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1338    }
1339
1340    #[serial(cooldown_state)]
1341    #[test]
1342    fn parse_review_response_from_markdown() {
1343        let md = r#"Here is my review:
1344
1345```json
1346{"decision":"request_changes","notes":"Fix the bug","issues":[{"file":"src/main.rs","line":10,"severity":"error","description":"null deref"}]}
1347```
1348
1349That's all."#;
1350        let resp = parse_review_response(md).unwrap();
1351        assert_eq!(resp.decision, "request_changes");
1352        assert_eq!(resp.issues.len(), 1);
1353        assert_eq!(resp.issues[0].file, "src/main.rs");
1354    }
1355
1356    #[serial(cooldown_state)]
1357    #[test]
1358    fn parse_review_response_embedded_json_object() {
1359        let text = "Review complete.\n\n{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}\nThanks!";
1360        let resp = parse_review_response(text).unwrap();
1361        assert_eq!(resp.decision, "approve");
1362        assert_eq!(resp.notes, "LGTM");
1363    }
1364
1365    #[serial(cooldown_state)]
1366    #[test]
1367    fn parse_review_response_template_echo_infers_from_notes() {
1368        let json = r#"{"decision":"approve|request_changes","notes":"LGTM","issues":[]}"#;
1369        assert_eq!(parse_review_response(json).unwrap().decision, "approve");
1370
1371        let no_signal = r#"{"decision":"approve|request_changes","notes":"hmm","issues":[]}"#;
1372        assert!(parse_review_response(no_signal).is_err());
1373    }
1374
1375    #[serial(cooldown_state)]
1376    #[test]
1377    fn parse_review_response_invalid() {
1378        let result = parse_review_response("not json at all");
1379        assert!(result.is_err());
1380    }
1381
1382    #[serial(cooldown_state)]
1383    #[test]
1384    fn extract_json_block_basic() {
1385        let md = "text\n```json\n{\"key\": \"value\"}\n```\nmore";
1386        let result = extract_json_block(md);
1387        assert!(result.is_some());
1388        assert!(result.unwrap().contains("\"key\""));
1389    }
1390
1391    #[serial(cooldown_state)]
1392    #[test]
1393    fn extract_json_block_skips_non_json_blocks() {
1394        let md = "```json\nnot-json-array\n```\n\n```json\n{\"real\": true}\n```";
1395        let result = extract_json_block(md);
1396        assert!(result.is_some());
1397        assert!(result.unwrap().contains("\"real\""));
1398    }
1399
1400    #[serial(cooldown_state)]
1401    #[test]
1402    fn extract_json_block_none_when_missing() {
1403        assert!(extract_json_block("no code blocks here").is_none());
1404    }
1405
1406    #[serial(cooldown_state)]
1407    #[test]
1408    fn extract_json_block_skips_malformed_intermediate_blocks() {
1409        let md = "prefix ```json{\"broken\": true}\n\n```json\n{\"real\": true}\n```";
1410        let result = extract_json_block(md);
1411        assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1412    }
1413
1414    #[serial(cooldown_state)]
1415    #[test]
1416    fn extract_json_block_skips_multiple_malformed_intermediate_blocks() {
1417        let md = "prefix ```json{\"broken\": true}\nmid ```json[1, 2, 3]\n\n```json\n{\"real\": true}\n```";
1418        let result = extract_json_block(md);
1419        assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1420    }
1421
1422    // ── parse_review_response + infer_review_response (production path) ─
1423
1424    /// Helper that mirrors the production plain-text path in review.rs:
1425    /// parse_review_response → infer_review_response.
1426    fn parse_review_plain(text: &str) -> anyhow::Result<ReviewResponse> {
1427        parse_review_response(text).or_else(|_| {
1428            infer_review_response(text)
1429                .ok_or_else(|| anyhow::anyhow!("failed to parse review response"))
1430        })
1431    }
1432
1433    #[serial(cooldown_state)]
1434    #[test]
1435    fn parse_review_from_output_direct_json() {
1436        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1437        let resp = parse_review_response(json).unwrap();
1438        assert_eq!(resp.decision, "approve");
1439    }
1440
1441    #[serial(cooldown_state)]
1442    #[test]
1443    fn parse_review_from_output_markdown() {
1444        let md = "Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}\n```\n";
1445        let resp = parse_review_response(md).unwrap();
1446        assert_eq!(resp.decision, "request_changes");
1447    }
1448
1449    #[serial(cooldown_state)]
1450    #[test]
1451    fn parse_review_from_output_plain_text_fallback() {
1452        let text = "The review is done. LGTM.";
1453        let resp = parse_review_plain(text).unwrap();
1454        assert_eq!(resp.decision, "approve");
1455        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1456    }
1457
1458    /// opencode NDJSON stream — text events use the `part.text` format.
1459    #[serial(cooldown_state)]
1460    #[test]
1461    fn parse_review_from_output_ndjson_part_format() {
1462        let ndjson = concat!(
1463            r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1464            "\n",
1465            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Reviewing the PR...\n\n```json\n{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1466            "\n",
1467            r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop"}}"#,
1468        );
1469        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1470        assert_eq!(resp.decision, "approve");
1471        assert_eq!(resp.notes, "All checks pass");
1472    }
1473
1474    /// opencode NDJSON stream — text events use the newer `text` directly-in-event format.
1475    #[serial(cooldown_state)]
1476    #[test]
1477    fn parse_review_from_output_ndjson_direct_text_format() {
1478        let ndjson = concat!(
1479            r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1480            "\n",
1481            r#"{"type":"text","timestamp":1001,"sessionID":"ses_abc","text":"Here is my review:\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the bug\",\"test_results\":\"fail\",\"issues\":[]}\n```\n"}"#,
1482            "\n",
1483            r#"{"type":"step_finish","timestamp":1002,"sessionID":"ses_abc"}"#,
1484        );
1485        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1486        assert_eq!(resp.decision, "request_changes");
1487        assert_eq!(resp.notes, "Fix the bug");
1488    }
1489
1490    /// opencode NDJSON with ReviewResponse JSON directly in a text event (no markdown wrapper).
1491    #[serial(cooldown_state)]
1492    #[test]
1493    fn parse_review_from_output_ndjson_direct_json_in_text_event() {
1494        let ndjson = concat!(
1495            r#"{"type":"step_start","timestamp":1000}"#,
1496            "\n",
1497            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1498            "\n",
1499            r#"{"type":"step_finish","timestamp":1002}"#,
1500        );
1501        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1502        assert_eq!(resp.decision, "approve");
1503    }
1504
1505    #[serial(cooldown_state)]
1506    #[test]
1507    fn parse_review_from_output_ndjson_prefers_final_json_text_event() {
1508        let ndjson = concat!(
1509            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Working on it..."}}"#,
1510            "\n",
1511            r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1512        );
1513        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1514        assert_eq!(resp.decision, "approve");
1515        assert_eq!(resp.notes, "LGTM");
1516    }
1517
1518    /// Concatenated text events produce a valid ReviewResponse.
1519    #[serial(cooldown_state)]
1520    #[test]
1521    fn parse_review_from_output_ndjson_concatenated_text() {
1522        let ndjson = concat!(
1523            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Here is my\n"}}"#,
1524            "\n",
1525            r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"review:\n\n```json\n{\"decision\":\"approve\",\"notes\":\"OK\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1526        );
1527        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1528        assert_eq!(resp.decision, "approve");
1529    }
1530
1531    /// codex NDJSON stream — agent_message item carries the ReviewResponse JSON.
1532    ///
1533    /// Real format from codex `exec --json` output (observed 2026-03-06):
1534    /// thread.started / turn.started / item.completed(reasoning) / item.completed(agent_message) / turn.completed
1535    #[serial(cooldown_state)]
1536    #[test]
1537    fn parse_review_from_output_codex_agent_message() {
1538        let ndjson = concat!(
1539            r#"{"type":"thread.started","thread_id":"t1"}"#,
1540            "\n",
1541            r#"{"type":"turn.started"}"#,
1542            "\n",
1543            r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Planning git commands..."}}"#,
1544            "\n",
1545            r#"{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"git log --oneline -5","output":"abc123 fix: something"}}"#,
1546            "\n",
1547            r#"{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1548            "\n",
1549            r#"{"type":"turn.completed"}"#,
1550        );
1551        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1552        assert_eq!(resp.decision, "approve");
1553        assert_eq!(resp.notes, "All checks pass");
1554    }
1555
1556    /// codex NDJSON with ReviewResponse in a markdown code block inside agent_message.
1557    #[serial(cooldown_state)]
1558    #[test]
1559    fn parse_review_from_output_codex_agent_message_markdown() {
1560        let ndjson = concat!(
1561            r#"{"type":"thread.started","thread_id":"t1"}"#,
1562            "\n",
1563            r#"{"type":"turn.started"}"#,
1564            "\n",
1565            r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Reviewing..."}}"#,
1566            "\n",
1567            r#"{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the tests\",\"issues\":[]}\n```\n"}}"#,
1568            "\n",
1569            r#"{"type":"turn.completed"}"#,
1570        );
1571        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1572        assert_eq!(resp.decision, "request_changes");
1573        assert_eq!(resp.notes, "Fix the tests");
1574    }
1575
1576    /// codex reasoning items must NOT be extracted as review text — only agent_message is used.
1577    #[serial(cooldown_state)]
1578    #[test]
1579    fn codex_skips_reasoning_items() {
1580        let ndjson = concat!(
1581            r#"{"type":"item.completed","item":{"type":"reasoning","text":"internal thoughts..."}}"#,
1582            "\n",
1583            r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}}"#,
1584        );
1585        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1586        assert_eq!(resp.decision, "approve");
1587        assert_eq!(resp.notes, "LGTM");
1588    }
1589
1590    /// Negation patterns must NOT trigger a false approval.
1591    #[serial(cooldown_state)]
1592    #[test]
1593    fn infer_review_response_negation_not_approved() {
1594        assert!(
1595            parse_review_plain("The changes were not approved.").is_err(),
1596            "\"not approved\" should not infer approval"
1597        );
1598    }
1599
1600    #[serial(cooldown_state)]
1601    #[test]
1602    fn infer_review_response_negation_unapproved() {
1603        assert!(
1604            parse_review_plain("Unapproved changes found in the diff.").is_err(),
1605            "\"unapproved\" should not infer approval"
1606        );
1607    }
1608
1609    #[serial(cooldown_state)]
1610    #[test]
1611    fn infer_review_response_negation_not_approving() {
1612        assert!(
1613            parse_review_plain("I am not approving this PR.").is_err(),
1614            "\"not approving\" should not infer approval"
1615        );
1616    }
1617
1618    #[serial(cooldown_state)]
1619    #[test]
1620    fn infer_review_response_negation_cannot_be_approved() {
1621        assert!(
1622            parse_review_plain("This PR cannot be approved until the race in tick.rs is fixed.")
1623                .is_err(),
1624            "\"cannot be approved\" should not infer approval"
1625        );
1626    }
1627
1628    #[serial(cooldown_state)]
1629    #[test]
1630    fn infer_review_response_negation_should_not_be_approved() {
1631        assert!(
1632            parse_review_plain("The diff breaks the build and should not be approved.").is_err(),
1633            "\"should not be approved\" should not infer approval"
1634        );
1635    }
1636
1637    #[serial(cooldown_state)]
1638    #[test]
1639    fn infer_review_response_negation_not_yet_approved() {
1640        assert!(
1641            parse_review_plain("This change is not yet approved.").is_err(),
1642            "\"not yet approved\" should not infer approval"
1643        );
1644    }
1645
1646    #[serial(cooldown_state)]
1647    #[test]
1648    fn infer_review_response_negation_wont_approve() {
1649        assert!(
1650            parse_review_plain("I won't approve this until tests cover the retry path.").is_err(),
1651            "\"won't approve\" should not infer approval"
1652        );
1653    }
1654
1655    #[serial(cooldown_state)]
1656    #[test]
1657    fn infer_review_response_bare_approved() {
1658        assert!(
1659            parse_review_plain("The PR was approved.").is_err(),
1660            "bare \"approved\" should not infer approval"
1661        );
1662    }
1663
1664    /// Positive plain-text approval signals must still work.
1665    #[serial(cooldown_state)]
1666    #[test]
1667    fn infer_review_response_lgtm() {
1668        let resp = parse_review_plain("LGTM, everything looks fine.").unwrap();
1669        assert_eq!(resp.decision, "approve");
1670    }
1671
1672    /// Evidence phrases are not a decision token
1673    #[serial(cooldown_state)]
1674    #[test]
1675    fn infer_review_response_looks_good() {
1676        assert!(
1677            parse_review_plain("Looks good to me!").is_err(),
1678            "\"looks good\" should not infer approval"
1679        );
1680    }
1681
1682    #[serial(cooldown_state)]
1683    #[test]
1684    fn infer_review_response_all_checks_passed() {
1685        assert!(
1686            parse_review_plain(
1687                "The background task completed but I already got the CI results directly. All checks passed — no action needed.",
1688            )
1689            .is_err(),
1690            "\"all checks passed\" should not infer approval"
1691        );
1692    }
1693
1694    #[serial(cooldown_state)]
1695    #[test]
1696    fn infer_review_response_checks_passed() {
1697        assert!(
1698            parse_review_plain("All checks passed (fmt ✓, clippy ✓, tests ✓).").is_err(),
1699            "\"all checks passed\" should not infer approval"
1700        );
1701    }
1702
1703    #[serial(cooldown_state)]
1704    #[test]
1705    fn infer_review_response_all_tests_passed() {
1706        assert!(
1707            parse_review_plain("All tests passed and the implementation looks correct.").is_err(),
1708            "\"all tests passed\" should not infer approval"
1709        );
1710    }
1711
1712    #[serial(cooldown_state)]
1713    #[test]
1714    fn infer_review_response_no_issues_found() {
1715        assert!(
1716            parse_review_plain("Review complete. No issues found in the diff.").is_err(),
1717            "\"no issues found\" should not infer approval"
1718        );
1719    }
1720
1721    /// Partial "tests passed" mid-sentence must NOT infer approval without "all" prefix.
1722    #[serial(cooldown_state)]
1723    #[test]
1724    fn infer_review_response_partial_tests_passed_no_approve() {
1725        let text = "The existing tests passed before this change but the new payment logic has a null-pointer risk on line 42.";
1726        assert!(parse_review_plain(text).is_err());
1727    }
1728
1729    /// Partial "checks passed" mid-sentence must NOT infer approval without "all" prefix.
1730    #[serial(cooldown_state)]
1731    #[test]
1732    fn infer_review_response_partial_checks_passed_no_approve() {
1733        let text = "CI checks passed for the base branch but this PR adds a broken path.";
1734        assert!(parse_review_plain(text).is_err());
1735    }
1736
1737    /// LLMs sometimes echo the JSON decision field value literally: "Decision is `approve`."
1738    #[serial(cooldown_state)]
1739    #[test]
1740    fn infer_review_response_decision_is_approve_backtick() {
1741        let text = "Already retrieved the output and completed the review above. The CI test check passed (3m11s) and all other checks are green. Decision is `approve`.";
1742        let resp = parse_review_plain(text).unwrap();
1743        assert_eq!(resp.decision, "approve");
1744    }
1745
1746    /// "Decision: approve" without backticks should also match.
1747    #[serial(cooldown_state)]
1748    #[test]
1749    fn infer_review_response_decision_colon_approve() {
1750        let text = "Reviewed the PR. No blocking issues found. Decision: approve";
1751        let resp = parse_review_plain(text).unwrap();
1752        assert_eq!(resp.decision, "approve");
1753    }
1754
1755    /// Bare backtick `approve` in a negative context must NOT infer approval.
1756    /// e.g. "do not `approve`" previously matched the removed `lower.contains("`approve`")` check.
1757    #[serial(cooldown_state)]
1758    #[test]
1759    fn infer_review_response_negation_do_not_approve_backtick() {
1760        let text = "There are unresolved issues — do not `approve` this PR until they are fixed.";
1761        assert!(
1762            parse_review_plain(text).is_err(),
1763            "negative context with backtick approve must not infer approval"
1764        );
1765    }
1766
1767    // ── review parsing via find_agent_result + parse_review_response ────
1768
1769    /// Helper that mirrors the production path in review.rs:
1770    /// find_agent_result → parse_review_response → infer_review_response.
1771    fn parse_via_agent_path(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
1772        let text = find_agent_result(agent, output)
1773            .map(|r| r.result_text)
1774            .filter(|t| !t.is_empty())
1775            .unwrap_or_else(|| output.to_string());
1776        if let Ok(r) = parse_review_response(&text) {
1777            return Ok(r);
1778        }
1779        infer_review_response_from_text(&text)
1780            .ok_or_else(|| anyhow::anyhow!("failed to parse review response from {agent} output"))
1781    }
1782
1783    #[serial(cooldown_state)]
1784    #[test]
1785    fn agent_output_claude_ndjson_review() {
1786        let ndjson = concat!(
1787            r#"{"type":"system","subtype":"init","session_id":"s1"}"#,
1788            "\n",
1789            r#"{"type":"assistant","message":{"content":[{"type":"text","text":"Reviewing..."}]}}"#,
1790            "\n",
1791            r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}","usage":{"input_tokens":100,"output_tokens":20}}"#,
1792        );
1793        let resp = parse_via_agent_path("claude", ndjson).unwrap();
1794        assert_eq!(resp.decision, "approve");
1795    }
1796
1797    #[serial(cooldown_state)]
1798    #[test]
1799    fn agent_output_opencode_ndjson_review() {
1800        let ndjson = concat!(
1801            r#"{"type":"step_start","timestamp":1000}"#,
1802            "\n",
1803            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}"}}"#,
1804            "\n",
1805            r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop","tokens":{"input":100,"output":10}}}"#,
1806        );
1807        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1808        assert_eq!(resp.decision, "request_changes");
1809    }
1810
1811    #[serial(cooldown_state)]
1812    #[test]
1813    fn agent_output_codex_ndjson_review() {
1814        let ndjson = concat!(
1815            r#"{"type":"thread.started","thread_id":"t1"}"#,
1816            "\n",
1817            r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All good\",\"issues\":[]}"}}"#,
1818            "\n",
1819            r#"{"type":"turn.completed"}"#,
1820        );
1821        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1822        assert_eq!(resp.decision, "approve");
1823    }
1824
1825    #[serial(cooldown_state)]
1826    #[test]
1827    fn agent_output_plain_text_keyword_fallback() {
1828        let text = "All checks passed, LGTM.";
1829        let resp = parse_via_agent_path("claude", text).unwrap();
1830        assert_eq!(resp.decision, "approve");
1831    }
1832
1833    #[serial(cooldown_state)]
1834    #[test]
1835    fn agent_output_direct_json() {
1836        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1837        let resp = parse_via_agent_path("opencode", json).unwrap();
1838        assert_eq!(resp.decision, "approve");
1839    }
1840
1841    /// Regression: plain text containing JSON fragments should NOT trigger false
1842    /// positives. The per-agent extractor returns None for non-NDJSON text,
1843    /// so we fall through to keyword inference instead of generic NDJSON parsing.
1844    #[serial(cooldown_state)]
1845    #[test]
1846    fn agent_output_text_with_json_fragment_no_false_positive() {
1847        let text = r#"I checked the file and found {"type":"text"} in the output. No issues."#;
1848        // This should NOT extract a review from the JSON fragment.
1849        // With the generic parser, this would match as a "text" NDJSON event.
1850        // With per-agent extraction, it returns None (not valid agent NDJSON).
1851        let result = parse_via_agent_path("claude", text);
1852        // Should either fail or infer from keywords, NOT parse the JSON fragment
1853        assert!(
1854            result.is_err() || result.as_ref().is_ok_and(|r| r.decision != "text"),
1855            "should not extract review from embedded JSON fragment"
1856        );
1857    }
1858
1859    /// Prose containing the word "decision" and a JSON-like fragment must not
1860    /// be misidentified as a review response by `extract_review_response_object`.
1861    #[serial(cooldown_state)]
1862    #[test]
1863    fn parse_review_response_ignores_prose_with_decision_word() {
1864        let text = r#"The decision was made. Here is a debug trace: {"trace":123}"#;
1865        assert!(
1866            parse_review_response(text).is_err(),
1867            "prose with 'decision' word and unrelated JSON should not parse as ReviewResponse"
1868        );
1869    }
1870
1871    /// Plain text with an embedded JSON object that has unrelated keys should
1872    /// not be extracted as a review response.
1873    #[serial(cooldown_state)]
1874    #[test]
1875    fn parse_review_response_ignores_embedded_json_without_decision_key() {
1876        let text = r#"Analysis complete. Config: {"timeout":30,"retries":3}. Done."#;
1877        assert!(
1878            parse_review_response(text).is_err(),
1879            "embedded JSON without 'decision' key should not parse as ReviewResponse"
1880        );
1881    }
1882
1883    // ── read_file_with_limit tests ───────────────────────────────────────────
1884
1885    #[serial(cooldown_state)]
1886    #[test]
1887    fn read_file_with_limit_reads_small_file_completely() {
1888        let temp_dir = env::temp_dir();
1889        let file_path = temp_dir.join(format!("orch_test_small_{}.txt", process::id()));
1890        let content = "Hello, World!";
1891        fs::write(&file_path, content).unwrap();
1892
1893        let result = read_file_with_limit(&file_path, 1024);
1894        assert!(result.is_ok());
1895        assert_eq!(result.unwrap(), content);
1896
1897        let _ = fs::remove_file(&file_path);
1898    }
1899
1900    #[serial(cooldown_state)]
1901    #[test]
1902    fn read_file_with_limit_truncates_large_file() {
1903        let temp_dir = env::temp_dir();
1904        let file_path = temp_dir.join(format!("orch_test_large_{}.txt", process::id()));
1905
1906        // Create a file larger than the limit (100 bytes)
1907        let large_content = "x".repeat(200);
1908        fs::write(&file_path, &large_content).unwrap();
1909
1910        // Read with a 50-byte limit
1911        let result = read_file_with_limit(&file_path, 50);
1912        assert!(result.is_ok());
1913
1914        let read_content = result.unwrap();
1915        // Should only read 50 bytes (the tail of the file)
1916        assert_eq!(read_content.len(), 50);
1917        assert_eq!(read_content, "x".repeat(50));
1918
1919        let _ = fs::remove_file(&file_path);
1920    }
1921
1922    #[serial(cooldown_state)]
1923    #[test]
1924    fn read_file_with_limit_handles_empty_file() {
1925        let temp_dir = env::temp_dir();
1926        let file_path = temp_dir.join(format!("orch_test_empty_{}.txt", process::id()));
1927        fs::write(&file_path, "").unwrap();
1928
1929        let result = read_file_with_limit(&file_path, 1024);
1930        assert!(result.is_ok());
1931        assert_eq!(result.unwrap(), "");
1932
1933        let _ = fs::remove_file(&file_path);
1934    }
1935
1936    #[serial(cooldown_state)]
1937    #[test]
1938    fn read_file_with_limit_handles_nonexistent_file() {
1939        let temp_dir = env::temp_dir();
1940        let file_path = temp_dir.join("orch_test_nonexistent_xyz123.txt");
1941
1942        let result = read_file_with_limit(&file_path, 1024);
1943        assert!(result.is_err());
1944    }
1945
1946    #[serial(cooldown_state)]
1947    #[test]
1948    fn read_file_with_limit_reads_exact_limit() {
1949        let temp_dir = env::temp_dir();
1950        let file_path = temp_dir.join(format!("orch_test_exact_{}.txt", process::id()));
1951
1952        // Create exactly 100 bytes
1953        let content = "y".repeat(100);
1954        fs::write(&file_path, &content).unwrap();
1955
1956        // Read with exactly 100-byte limit (should read all)
1957        let result = read_file_with_limit(&file_path, 100);
1958        assert!(result.is_ok());
1959        assert_eq!(result.unwrap().len(), 100);
1960
1961        let _ = fs::remove_file(&file_path);
1962    }
1963
1964    #[serial(cooldown_state)]
1965    #[test]
1966    fn read_file_with_limit_handles_unicode_truncation() {
1967        let temp_dir = env::temp_dir();
1968        let file_path = temp_dir.join(format!("orch_test_unicode_{}.txt", process::id()));
1969
1970        // Create content with multi-byte UTF-8 characters
1971        let emoji = "🎉".repeat(100); // Each emoji is 4 bytes
1972        fs::write(&file_path, &emoji).unwrap();
1973
1974        // Read with a limit that cuts into the emoji sequence
1975        let result = read_file_with_limit(&file_path, 50);
1976        assert!(result.is_ok());
1977
1978        let read_content = result.unwrap();
1979        // from_utf8_lossy should handle invalid sequences
1980        assert_ne!(read_content, "");
1981
1982        let _ = fs::remove_file(&file_path);
1983    }
1984
1985    // ── Fixture-based review integration tests ──────────────────────────────────
1986
1987    /// Full review pipeline: find_agent_result → parse_review_response → infer_review_response.
1988    fn parse_review_via_agent(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
1989        let text = find_agent_result(agent, output)
1990            .map(|r| r.result_text)
1991            .filter(|t| !t.is_empty())
1992            .unwrap_or_else(|| output.to_string());
1993        if let Ok(r) = parse_review_response(&text) {
1994            return Ok(r);
1995        }
1996        infer_review_response(&text)
1997            .ok_or_else(|| anyhow::anyhow!("failed to parse review from {agent} output"))
1998    }
1999
2000    // ── Claude review fixtures ─────────────────────────────────────────────────
2001
2002    #[serial(cooldown_state)]
2003    #[test]
2004    fn fixture_claude_review_approve() {
2005        let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2006        let resp = parse_review_via_agent("claude", raw).unwrap();
2007        assert_eq!(resp.decision, "approve");
2008        assert_eq!(resp.notes, "All checks pass");
2009        assert_eq!(resp.test_results.as_deref(), Some("pass"));
2010        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2011    }
2012
2013    #[serial(cooldown_state)]
2014    #[test]
2015    fn fixture_claude_review_request_changes() {
2016        let raw = include_str!("../../../tests/fixtures/review_claude_request_changes.jsonl");
2017        let resp = parse_review_via_agent("claude", raw).unwrap();
2018        assert_eq!(resp.decision, "request_changes");
2019        assert_eq!(resp.notes, "Fix the null deref on line 42");
2020        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2021        assert_eq!(resp.issues.len(), 1);
2022        assert_eq!(resp.issues[0].file, "src/main.rs");
2023        assert_eq!(resp.issues[0].line, Some(42));
2024        assert_eq!(resp.issues[0].severity, "error");
2025    }
2026
2027    #[serial(cooldown_state)]
2028    #[test]
2029    fn fixture_claude_review_rate_limit_returns_error() {
2030        // find_claude_result on an is_error=true result returns Some with is_error=true.
2031        // The integration test verifies this is captured, not that it's a ReviewResponse.
2032        let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2033        let result = find_agent_result("claude", raw);
2034        assert!(
2035            result.is_some(),
2036            "should find result even for error envelope"
2037        );
2038        let result = result.unwrap();
2039        assert!(result.is_error, "rate limit should be flagged as error");
2040        assert!(result.result_text.contains("rate limit"));
2041    }
2042
2043    // ── OpenCode review fixtures ────────────────────────────────────────────────
2044
2045    #[serial(cooldown_state)]
2046    #[test]
2047    fn fixture_opencode_review_approve() {
2048        let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2049        let resp = parse_review_via_agent("opencode", raw).unwrap();
2050        assert_eq!(resp.decision, "approve");
2051        assert_eq!(resp.notes, "All checks pass");
2052        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2053    }
2054
2055    #[serial(cooldown_state)]
2056    #[test]
2057    fn fixture_opencode_review_request_changes() {
2058        let raw = include_str!("../../../tests/fixtures/review_opencode_request_changes.jsonl");
2059        let resp = parse_review_via_agent("opencode", raw).unwrap();
2060        assert_eq!(resp.decision, "request_changes");
2061        assert_eq!(resp.notes, "Fix the memory leak");
2062        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2063        assert_eq!(resp.issues.len(), 1);
2064        assert_eq!(resp.issues[0].file, "src/lib.rs");
2065        assert_eq!(resp.issues[0].line, Some(100));
2066    }
2067
2068    #[serial(cooldown_state)]
2069    #[test]
2070    fn fixture_opencode_review_plain_text_approve() {
2071        // OpenCode with plain-text review (no JSON) — should infer approval via keywords.
2072        let raw = include_str!("../../../tests/fixtures/review_opencode_plain_text.jsonl");
2073        let resp = parse_review_via_agent("opencode", raw).unwrap();
2074        assert_eq!(resp.decision, "approve");
2075    }
2076
2077    // ── Codex review fixtures ────────────────────────────────────────────────────
2078
2079    #[serial(cooldown_state)]
2080    #[test]
2081    fn fixture_codex_review_approve() {
2082        let raw = include_str!("../../../tests/fixtures/review_codex_approve.jsonl");
2083        let resp = parse_review_via_agent("codex", raw).unwrap();
2084        assert_eq!(resp.decision, "approve");
2085        assert_eq!(resp.notes, "Clean implementation");
2086        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2087    }
2088
2089    #[serial(cooldown_state)]
2090    #[test]
2091    fn fixture_codex_review_request_changes() {
2092        let raw = include_str!("../../../tests/fixtures/review_codex_request_changes.jsonl");
2093        let resp = parse_review_via_agent("codex", raw).unwrap();
2094        assert_eq!(resp.decision, "request_changes");
2095        assert_eq!(resp.notes, "Race condition on line 15");
2096        assert_eq!(resp.issues.len(), 1);
2097        assert_eq!(resp.issues[0].file, "src/server.rs");
2098        assert_eq!(resp.issues[0].line, Some(15));
2099    }
2100
2101    #[serial(cooldown_state)]
2102    #[test]
2103    fn fixture_codex_review_plain_text_approve() {
2104        // Codex with plain-text review — should infer approval via "LGTM" keyword.
2105        let raw = include_str!("../../../tests/fixtures/review_codex_plain_text.jsonl");
2106        let resp = parse_review_via_agent("codex", raw).unwrap();
2107        assert_eq!(resp.decision, "approve");
2108    }
2109
2110    // ── Kimi review fixtures ────────────────────────────────────────────────────
2111
2112    #[serial(cooldown_state)]
2113    #[test]
2114    fn fixture_kimi_review_approve() {
2115        let raw = include_str!("../../../tests/fixtures/review_kimi_approve.jsonl");
2116        let resp = parse_review_via_agent("kimi", raw).unwrap();
2117        assert_eq!(resp.decision, "approve");
2118        assert_eq!(resp.notes, "LGTM");
2119        assert_eq!(resp.test_results.as_deref(), Some("pass"));
2120        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2121    }
2122
2123    // ── MiniMax review fixtures ────────────────────────────────────────────────
2124
2125    #[serial(cooldown_state)]
2126    #[test]
2127    fn fixture_minimax_review_request_changes() {
2128        let raw = include_str!("../../../tests/fixtures/review_minimax_request_changes.jsonl");
2129        let resp = parse_review_via_agent("minimax", raw).unwrap();
2130        assert_eq!(resp.decision, "request_changes");
2131        assert_eq!(resp.notes, "Fix the bug");
2132        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2133        assert_eq!(resp.issues.len(), 1);
2134        assert_eq!(resp.issues[0].file, "src/foo.rs");
2135        assert_eq!(resp.issues[0].line, Some(42));
2136        assert_eq!(resp.issues[0].severity, "error");
2137    }
2138
2139    // ── Cross-agent token/metadata extraction ───────────────────────────────────
2140
2141    /// Verify that find_agent_result extracts token counts from each agent's format.
2142    #[serial(cooldown_state)]
2143    #[test]
2144    fn find_agent_result_extracts_tokens_claude() {
2145        let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2146        let result = find_agent_result("claude", raw).expect("should find result");
2147        assert_eq!(result.input_tokens, Some(200));
2148        assert_eq!(result.output_tokens, Some(30));
2149        assert!(result.cost_usd.is_none()); // review fixtures don't include cost
2150    }
2151
2152    #[serial(cooldown_state)]
2153    #[test]
2154    fn find_agent_result_extracts_tokens_opencode() {
2155        let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2156        let result = find_agent_result("opencode", raw).expect("should find result");
2157        assert_eq!(result.input_tokens, Some(2900));
2158        assert_eq!(result.output_tokens, Some(100));
2159        assert_eq!(result.cost_usd, Some(0.002));
2160    }
2161
2162    /// Verify that find_agent_result returns None for empty/unparseable input.
2163    #[serial(cooldown_state)]
2164    #[test]
2165    fn find_agent_result_returns_none_for_empty() {
2166        assert!(find_agent_result("claude", "").is_none());
2167        assert!(find_agent_result("opencode", "").is_none());
2168        assert!(find_agent_result("codex", "").is_none());
2169        assert!(find_agent_result("kimi", "").is_none());
2170        assert!(find_agent_result("minimax", "").is_none());
2171    }
2172
2173    #[serial(cooldown_state)]
2174    #[test]
2175    fn find_agent_result_returns_none_for_plain_text() {
2176        assert!(find_agent_result("claude", "just some plain text").is_none());
2177        assert!(find_agent_result("codex", "just some plain text").is_none());
2178        // OpenCode is a special case — it can sometimes return a result for plain text.
2179        // This is acceptable because opencode's extract_ndjson_text falls back gracefully.
2180    }
2181
2182    // ── Error propagation via find_agent_result ─────────────────────────────────
2183
2184    /// Verify that find_agent_result surfaces rate-limit errors from Claude.
2185    #[serial(cooldown_state)]
2186    #[test]
2187    fn find_agent_result_propagates_claude_rate_limit() {
2188        let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2189        let result = find_agent_result("claude", raw).expect("should find result");
2190        assert!(result.is_error, "should be flagged as error");
2191        assert!(result.result_text.contains("rate limit"));
2192    }
2193
2194    /// Two fake agents on PATH, a config that names them and a 1 ms backoff. Each nextest test
2195    /// runs in its own process, so the environment is not restored
2196    #[cfg(unix)]
2197    fn failover_env() -> tempfile::TempDir {
2198        let dir = tempfile::tempdir().unwrap();
2199        let bin = dir.path().join("bin");
2200        fs::create_dir(&bin).unwrap();
2201        for agent in ["claude", "codex"] {
2202            let path = bin.join(agent);
2203            fs::write(&path, "#!/bin/sh\n").unwrap();
2204            fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap();
2205        }
2206        let home = dir.path().join("home");
2207        fs::create_dir(&home).unwrap();
2208        fs::write(
2209            home.join("config.yml"),
2210            "agents: [claude, codex]\ndispatch:\n  retry_base_delay_ms: 1\n  retry_max_delay_ms: 1\nworkflow:\n  max_timeout_retries: 3\n",
2211        )
2212        .unwrap();
2213        let mut path = bin.as_os_str().to_os_string();
2214        path.push(":");
2215        path.push(env::var_os("PATH").unwrap_or_default());
2216        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2217        unsafe { env::set_var("PATH", path) };
2218        // SAFETY: same as above
2219        unsafe { env::set_var("ORCH_HOME", &home) };
2220        cmd_cache::reset_for_test();
2221        dir
2222    }
2223
2224    async fn failover_task() -> (Arc<TaskStore>, String) {
2225        let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
2226        let id = store
2227            .create_internal("owner/repo", "t", "b", "test", "", None)
2228            .await
2229            .unwrap();
2230        (store, format!("internal:{id}"))
2231    }
2232
2233    /// Every other agent is in cooldown, so the timed-out task retries from `new` after the backoff
2234    #[cfg(unix)]
2235    #[serial(cooldown_state)]
2236    #[tokio::test]
2237    async fn timeout_with_every_other_agent_cooled_retries_as_new() {
2238        let _env = failover_env();
2239        cooldown::reset_global_state().await;
2240        set_agent_cooldown("codex", 600).await;
2241        let (store, task_id) = failover_task().await;
2242        let store = Some(store);
2243
2244        let status = handle_failover(
2245            &task_id,
2246            "claude",
2247            RetryableError::Timeout,
2248            "claude timed out after 1804s",
2249            false,
2250            &store,
2251            "owner/repo",
2252        )
2253        .await;
2254
2255        assert_eq!(status, "new");
2256        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2257            .await
2258            .unwrap();
2259        assert!(task.limit_reroute_chain.is_empty(), "chain must be cleared");
2260        assert!(
2261            task.last_error.contains("will retry"),
2262            "{}",
2263            task.last_error
2264        );
2265        assert_eq!(task.timeout_retries, 1);
2266    }
2267
2268    /// Another agent is free, so the timeout still reroutes to it
2269    #[cfg(unix)]
2270    #[serial(cooldown_state)]
2271    #[tokio::test]
2272    async fn timeout_with_a_free_agent_reroutes_to_it() {
2273        let _env = failover_env();
2274        cooldown::reset_global_state().await;
2275        let (store, task_id) = failover_task().await;
2276        let store = Some(store);
2277
2278        let status = handle_failover(
2279            &task_id,
2280            "claude",
2281            RetryableError::Timeout,
2282            "claude timed out after 1804s",
2283            false,
2284            &store,
2285            "owner/repo",
2286        )
2287        .await;
2288
2289        assert_eq!(status, "new");
2290        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2291            .await
2292            .unwrap();
2293        assert_eq!(task.limit_reroute_chain, "claude");
2294        assert!(task.last_error.contains("re-route"), "{}", task.last_error);
2295    }
2296
2297    /// A task that times out on every run blocks after `max_timeout_retries` and keeps its issue open
2298    #[cfg(unix)]
2299    #[serial(cooldown_state)]
2300    #[tokio::test]
2301    async fn repeated_timeouts_block_the_task() {
2302        let _env = failover_env();
2303        cooldown::reset_global_state().await;
2304        set_agent_cooldown("codex", 600).await;
2305        let (store, task_id) = failover_task().await;
2306        let store = Some(store);
2307
2308        let mut statuses = Vec::new();
2309        for _ in 0..3 {
2310            statuses.push(
2311                handle_failover(
2312                    &task_id,
2313                    "claude",
2314                    RetryableError::Timeout,
2315                    "claude timed out after 1804s",
2316                    false,
2317                    &store,
2318                    "owner/repo",
2319                )
2320                .await,
2321            );
2322        }
2323
2324        assert_eq!(statuses, ["new", "new", "blocked"]);
2325        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2326            .await
2327            .unwrap();
2328        assert_eq!(task.block_reason.as_deref(), Some("timed out 3 times"));
2329        assert_eq!(task.timeout_retries, 3);
2330    }
2331}