Skip to main content

orch/engine/runner/
response.rs

1//! Response collection + error classification.
2//!
3//! After the agent finishes (tmux session ends), this module:
4//! 1. Reads the output file
5//! 2. Parses the response JSON
6//! 3. Classifies errors (timeout, usage limit, auth, tooling)
7//! 4. Determines next action (success, reroute, needs_review)
8
9use crate::store;
10use crate::store::TaskStore;
11use std::collections::HashSet;
12use std::collections::hash_map::DefaultHasher;
13use std::fs;
14use std::fs::File;
15use std::hash::Hash;
16use std::hash::Hasher;
17use std::io::Read;
18use std::io::Seek;
19use std::io::SeekFrom;
20use std::path::{Path, PathBuf};
21use std::result;
22use std::sync::Arc;
23use std::thread;
24use std::time::Duration;
25use std::time::SystemTime;
26
27use tokio::fs as tokio_fs;
28use tokio::task;
29use tokio::time;
30// Cooldown tracking is implemented in `crate::engine::cooldown` (shared with the router).
31use crate::cli::ndjson;
32use crate::cmd_cache;
33use crate::config as app_config;
34use crate::engine;
35pub use crate::engine::cooldown::{
36    clear_expired_cooldowns, is_agent_in_cooldown, is_model_in_cooldown,
37    record_agent_failure_with_message, record_model_failure, set_agent_cooldown,
38};
39use crate::engine::router::config;
40use crate::home;
41use crate::parser::AgentResponse;
42use crate::store::MemoryEntry;
43use crate::store::Task;
44
45const JITTER_FACTOR: f64 = 0.3;
46
47/// Maximum output file size to read (10 MB).
48/// Prevents runaway outputs from consuming excessive memory.
49const MAX_OUTPUT_SIZE_BYTES: u64 = 10 * 1024 * 1024;
50
51/// Read a file with a size limit, returning truncated content if the file is too large.
52///
53/// This prevents runaway agent outputs from causing memory issues. If the file
54/// exceeds the limit, reads from the end of the file to capture the most recent
55/// output (which is typically where the result or error message is).
56fn read_file_with_limit<P: AsRef<Path>>(path: P, max_bytes: u64) -> anyhow::Result<String> {
57    let path = path.as_ref();
58    let metadata = fs::metadata(path)?;
59
60    if !metadata.is_file() {
61        anyhow::bail!("not a file: {}", path.display());
62    }
63
64    let file_size = metadata.len();
65
66    // If file is within limits, read the whole thing
67    if file_size <= max_bytes {
68        return fs::read_to_string(path).map_err(Into::into);
69    }
70
71    // File is too large - read from the end to get the most recent output
72    let mut file = File::open(path)?;
73    let start_pos = file_size.saturating_sub(max_bytes);
74    file.seek(SeekFrom::Start(start_pos))?;
75
76    let mut buffer = Vec::with_capacity(usize::try_from(max_bytes).unwrap_or(0));
77    file.read_to_end(&mut buffer)?;
78
79    // Try to convert to UTF-8, replacing invalid sequences
80    let content = String::from_utf8_lossy(&buffer);
81
82    // Log a warning about truncation
83    tracing::warn!(
84        path = %path.display(),
85        file_size = file_size,
86        max_bytes = max_bytes,
87        "output file truncated: exceeded size limit"
88    );
89
90    Ok(content.into_owned())
91}
92
93/// Calculate exponential backoff with jitter for fallback retries.
94/// Uses retry count from the reroute chain to determine the delay.
95///
96/// Delay = min(base * 2^retry_count, max) * (1 + jitter)
97pub(crate) fn calculate_backoff_delay(retry_count: usize) -> u64 {
98    let base_delay = config::retry_base_delay_ms();
99    let max_delay = config::retry_max_delay_ms();
100
101    // Exponential: base * 2^retry_count
102    let exponential =
103        base_delay * (2_u64.saturating_pow(u32::try_from(retry_count).unwrap_or(u32::MAX)));
104    let capped = exponential.min(max_delay);
105
106    // Add jitter: ±30% using a hash-based pseudo-random value.
107    // Mix system time (nanoseconds) with thread ID so concurrent retries on
108    // different threads get different delays even when called in the same
109    // microsecond, avoiding the thundering-herd problem.
110    #[expect(
111        clippy::cast_precision_loss,
112        clippy::cast_possible_truncation,
113        clippy::cast_sign_loss,
114        reason = "a jitter range in milliseconds, far below 2^52 and never negative"
115    )]
116    let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
117    let jitter = {
118        let mut h = DefaultHasher::new();
119        SystemTime::now().hash(&mut h);
120        thread::current().id().hash(&mut h);
121        h.finish() % (jitter_range * 2 + 1)
122    };
123
124    capped.saturating_sub(jitter_range) + jitter
125}
126
127fn fallback_retry_count(task: Option<&Task>) -> usize {
128    task.map_or(0, |t| {
129        let chain_retry_count = t
130            .limit_reroute_chain
131            .split(',')
132            .filter(|entry| !entry.is_empty())
133            .count();
134        let network_retry_count = usize::try_from(t.network_retries).unwrap_or(0);
135        chain_retry_count.max(network_retry_count)
136    })
137}
138
139/// Wait with exponential backoff before retrying with a fallback agent.
140/// Returns the delay applied in milliseconds.
141pub async fn wait_for_fallback_backoff(
142    task_id: &str,
143    store: &Option<Arc<TaskStore>>,
144    repo: &str,
145) -> u64 {
146    let task = store::opt_store_get_task(store, repo, task_id).await;
147
148    // Normal failover uses reroute chain length. NetworkError retries bypass
149    // failover, so track their own streak and use whichever retry source is larger.
150    let retry_count = fallback_retry_count(task.as_ref());
151
152    let delay = calculate_backoff_delay(retry_count);
153
154    if delay > 0 {
155        tracing::debug!(
156            task_id,
157            retry_count = retry_count,
158            delay_ms = delay,
159            "fallback backoff: waiting before retry"
160        );
161        time::sleep(Duration::from_millis(delay)).await;
162    }
163
164    delay
165}
166
167/// Outcome signal for the engine to update router weights.
168///
169/// Returned by the task runner so the engine can feed rate limit and
170/// success signals back to the router's weighted round-robin implementation.
171#[derive(Debug, Clone)]
172pub enum WeightSignal {
173    /// Agent completed a task successfully.
174    Success { agent: String },
175    /// Agent hit a rate limit / usage limit.
176    RateLimited { agent: String },
177    /// The run failed (timeout, no fallback, exhausted agents) and ended in `needs_review`.
178    /// Never read as a completed run, so the no-PR shortcut cannot mark the task done.
179    Failed { agent: String },
180    /// Task is blocked on child tasks (delegations). Do not trigger review agent.
181    Blocked,
182    /// Task was rerouted (silence detection, timeout, parse error) — reset to new for re-routing.
183    /// Distinct from None so the engine never converts a silent-reset task into needs_review.
184    Rerouted,
185    /// No weight-relevant signal (guard skipped task, unrecognised status, etc.)
186    None,
187}
188
189/// Read the agent's output file, trying multiple locations.
190///
191/// Checks per-task attempt directory first, then legacy flat paths.
192/// Output is truncated to `MAX_OUTPUT_SIZE_BYTES` to prevent runaway memory usage.
193pub async fn read_output_file(task_id: &str, primary_path: &Path, repo: &str) -> String {
194    // Primary: explicit output file (already points to attempt dir)
195    if let Some(content) = task::spawn_blocking({
196        let p = primary_path.to_path_buf();
197        move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
198    })
199    .await
200    .ok()
201    .and_then(result::Result::ok)
202        && !content.is_empty()
203    {
204        return content;
205    }
206
207    // Fallback: check all attempt dirs for this task (newest first)
208    if let Ok(task_dir) = home::task_dir_async(repo, task_id).await {
209        let attempts_dir = task_dir.join("attempts");
210        // Use async metadata for is_dir check
211        if tokio_fs::metadata(&attempts_dir)
212            .await
213            .is_ok_and(|m| m.is_dir())
214        {
215            let mut attempt_nums: Vec<u32> = task::spawn_blocking({
216                let a = attempts_dir.clone();
217                move || {
218                    match fs::read_dir(&a) {
219                        Ok(entries) => {
220                            let mut nums = Vec::new();
221                            for entry in entries {
222                                match entry {
223                                    Ok(e) => {
224                                        if let Some(n) =
225                                            e.file_name().to_str().and_then(|n| n.parse().ok())
226                                        {
227                                            nums.push(n);
228                                        }
229                                    }
230                                    Err(e) => tracing::warn!(
231                                        path = %a.display(),
232                                        err = %e,
233                                        "error reading attempt dir entry in output fallback"
234                                    ),
235                                }
236                            }
237                            nums
238                        }
239                        Err(e) => {
240                            tracing::warn!(path = %a.display(), err = %e, "failed to read attempts dir in output fallback");
241                            Vec::new()
242                        }
243                    }
244                }
245            })
246            .await
247            .unwrap_or_default();
248
249            attempt_nums.sort_unstable_by(|a, b| b.cmp(a)); // newest first
250            for n in attempt_nums {
251                let p = attempts_dir.join(n.to_string()).join("output.json");
252                if let Some(content) = task::spawn_blocking({
253                    let p = p.clone();
254                    move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
255                })
256                .await
257                .ok()
258                .and_then(result::Result::ok)
259                    && !content.is_empty()
260                {
261                    tracing::info!(task_id, path = %p.display(), "read output from attempt dir");
262                    return content;
263                }
264            }
265        }
266    }
267
268    // Legacy fallback locations
269    let state_dir = home::state_dir().unwrap_or_else(|_| PathBuf::from("/tmp"));
270
271    let mut fallbacks = vec![
272        PathBuf::from(format!("/tmp/output-{task_id}.json")),
273        state_dir.join(format!("output-{task_id}.json")),
274    ];
275
276    if let Ok(legacy_path) = home::state_file(&format!("output-{task_id}.json"))
277        && !fallbacks.contains(&legacy_path)
278    {
279        fallbacks.push(legacy_path);
280    }
281
282    for path in &fallbacks {
283        if let Some(content) = task::spawn_blocking({
284            let p = path.clone();
285            move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
286        })
287        .await
288        .ok()
289        .and_then(result::Result::ok)
290            && !content.is_empty()
291        {
292            tracing::info!(task_id, path = %path.display(), "read output from legacy fallback");
293            return content;
294        }
295    }
296
297    String::new()
298}
299
300/// Pick a fallback agent, avoiding agents already in the reroute chain and agents in cooldown.
301pub fn pick_fallback_agent(
302    current_agent: &str,
303    chain: &str,
304    available_agents: &[String],
305) -> Option<String> {
306    // Clear expired cooldowns first
307    clear_expired_cooldowns();
308
309    let chain_set: HashSet<&str> = if chain.is_empty() {
310        HashSet::new()
311    } else {
312        chain.split(',').collect()
313    };
314
315    // Prefer agents that are not in cooldown and are not heavily degraded
316    // from recent rate-limits. This prevents immediate failover to agents
317    // that have been heavily penalized and likely to fail as well.
318    for agent in available_agents {
319        if agent == current_agent || chain_set.contains(agent.as_str()) {
320            continue;
321        }
322        if is_agent_in_cooldown(agent) {
323            continue;
324        }
325        // Weight-based filtering was removed from this module because it used
326        // a default-constructed AgentWeights (dead code that always returned
327        // true). Keep failover behavior simple here: prefer the first agent
328        // that is not in cooldown and not in the reroute chain. If we later
329        // expose the router's AgentWeights snapshot to this module we can add
330        // weight-based avoidance back in a non-dead way.
331        return Some(agent.clone());
332    }
333
334    None
335}
336
337/// Retryable error types that should trigger agent failover.
338#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum RetryableError {
340    /// Agent timed out.
341    Timeout,
342    /// Run exited without a result envelope
343    Interrupted,
344    /// Usage/rate limit hit.
345    UsageLimit,
346    /// Auth/billing error.
347    AuthError,
348    /// General failure (non-retryable but we still try fallback agents).
349    Failed,
350    /// Missing tooling - fallback might help if another agent has the tool.
351    MissingTooling,
352    /// A specific model was unavailable (e.g. "Model not found"). The agent
353    /// itself is fine — only the model is in cooldown. Failover may still pick
354    /// a different agent, but we must not bump the agent-level failure counter.
355    ModelUnavailable,
356}
357
358impl RetryableError {
359    /// Return a short classified string for this error type.
360    /// Used in DB rate-limit metrics and test assertions.
361    pub fn type_str(self) -> &'static str {
362        match self {
363            RetryableError::Timeout => "timeout",
364            RetryableError::Interrupted => "interrupted",
365            RetryableError::UsageLimit => "rate_limit",
366            RetryableError::AuthError => "auth_error",
367            RetryableError::ModelUnavailable => "model_unavailable",
368            RetryableError::Failed | RetryableError::MissingTooling => "failed",
369        }
370    }
371}
372
373/// Next task status from [`handle_failover`], plus the fallback agent when one was stored
374#[derive(Debug, Clone, PartialEq, Eq)]
375pub struct FailoverOutcome {
376    /// Task status to apply: `new`, `blocked`, or `needs_review`
377    pub status: String,
378    /// Fallback agent written to the task when failover switched
379    pub next_agent: Option<String>,
380}
381
382impl FailoverOutcome {
383    fn with_status(status: impl Into<String>) -> Self {
384        Self {
385            status: status.into(),
386            next_agent: None,
387        }
388    }
389
390    fn switched(next_agent: String) -> Self {
391        Self {
392            status: "new".to_string(),
393            next_agent: Some(next_agent),
394        }
395    }
396}
397
398/**
399 * Handle failover for any retryable error type. Status is "new" on reroute (or
400 * retryable exhaustion), "blocked" on hard exhaustion, "needs_review" when no
401 * fallback agents are available. `next_agent` is set when a fallback was stored
402 * so the next route prefers it over an `agent:*` label. `credit_recorded` means
403 * the caller already ran `record_credit_exhaustion`; the generic `failure_count:`
404 * counter must not be bumped here or the next ordinary failure starts at the
405 * extended backoff tier (issue #3652). Rate-limit event DB recording is done by
406 * the caller (mod.rs)
407 **/
408pub async fn handle_failover(
409    task_id: &str,
410    agent_name: &str,
411    error_type: RetryableError,
412    error_message: &str,
413    credit_recorded: bool,
414    store: &Option<Arc<TaskStore>>,
415    repo: &str,
416) -> FailoverOutcome {
417    // Get the reroute chain
418    let chain = get_reroute_chain(task_id, store, repo).await;
419    let chain = update_reroute_chain(task_id, agent_name, &chain, store, repo).await;
420
421    if error_type == RetryableError::Timeout
422        && let Some(status) =
423            block_after_repeated_timeouts(task_id, error_message, store, repo).await
424    {
425        return FailoverOutcome::with_status(status);
426    }
427
428    // Get all available agents
429    let available: Vec<String> = engine::configured_agents()
430        .into_iter()
431        .filter(|a| cmd_cache::command_exists(a))
432        .collect();
433
434    // Check if we've exhausted all agents (build chain_set once)
435    let chain_set: HashSet<&str> = if chain.is_empty() {
436        HashSet::new()
437    } else {
438        chain.split(',').collect()
439    };
440    let is_exhausted = available.iter().all(|a| chain_set.contains(a.as_str()));
441
442    if is_exhausted {
443        // Retryable failures (rate limits, quota, model unavailable) will resolve once
444        // cooldowns expire — reset to "new" so the router picks it up again then.
445        // Hard failures (generic agent crash, missing tooling) won't improve on retry;
446        // block immediately with a clear reason rather than wasting 63 minutes in the
447        // review-refire loop before getting a misleading "review agent rebroadcast" message.
448        let is_retryable = matches!(
449            error_type,
450            RetryableError::UsageLimit
451                | RetryableError::AuthError
452                | RetryableError::ModelUnavailable
453                | RetryableError::Timeout
454                | RetryableError::Interrupted
455        );
456        let next_status = if is_retryable { "new" } else { "blocked" };
457
458        tracing::warn!(
459            task_id,
460            agent = agent_name,
461            error_type = ?error_type,
462            next_status,
463            "all agents exhausted"
464        );
465
466        let mut store_fields: Vec<(&str, serde_json::Value)> = Vec::new();
467
468        if is_retryable {
469            // Clear reroute chain so the next dispatch builds a fresh one rather than
470            // immediately seeing "all agents in chain" and exhausting again.
471            let msg = format!("{error_message} (all agents exhausted, will retry when available)");
472            store_fields.push(("last_error", serde_json::json!(msg)));
473            store_fields.push(("limit_reroute_chain", serde_json::json!("")));
474        } else {
475            let msg = format!("{error_message} (all agents exhausted)");
476            let block_reason = format!(
477                "all agents exhausted: {} — {}",
478                error_type.type_str(),
479                error_message
480            );
481            store_fields.push(("last_error", serde_json::json!(msg)));
482            store_fields.push(("block_reason", serde_json::json!(block_reason)));
483        }
484
485        if let Err(e) = store::store_set_result(store, repo, task_id, &store_fields).await {
486            tracing::warn!(task_id, err = %e, "failed to write agents_exhausted fields to store");
487        }
488
489        // Log reroute activity for exhausted agents
490        let details = serde_json::json!({
491            "failure_reason": error_type.type_str(),
492            "error_message": error_message,
493            "chain": chain,
494            "agents_exhausted": true,
495            "is_retryable": is_retryable,
496        });
497        store::store_log_activity(
498            store,
499            repo,
500            task_id,
501            "rerouted",
502            Some("in_progress"),
503            Some(next_status),
504            None::<&str>,
505            None::<&str>,
506            Some(&details),
507        )
508        .await;
509
510        return FailoverOutcome::with_status(next_status);
511    }
512
513    // Pick a fallback agent
514    if let Some(next) = pick_fallback_agent(agent_name, &chain, &available) {
515        tracing::info!(
516            task_id,
517            from = agent_name,
518            to = next,
519            error_type = ?error_type,
520            "failover: switching to fallback agent"
521        );
522
523        // Timeouts and interrupts are not agent faults, skip the agent cooldown
524        if !matches!(
525            error_type,
526            RetryableError::MissingTooling
527                | RetryableError::ModelUnavailable
528                | RetryableError::Timeout
529                | RetryableError::Interrupted
530        ) {
531            // Credit errors skip the generic counter (already counted via
532            // credit_failure_count); the 120s cooldown below still applies
533            if !credit_recorded {
534                record_agent_failure_with_message(agent_name, error_message).await;
535            }
536
537            // Apply brief 120s cooldown on the failed agent so the router skips it
538            // on the next routing attempt. This prevents the router from immediately
539            // re-selecting the same agent after clearing both agent and model (#1492).
540            // Also skipped for ModelUnavailable: the agent is healthy.
541            set_agent_cooldown(agent_name, 120).await;
542        }
543
544        // Store the fallback agent so the next route prefers it over an agent:* label
545        // Clear the model so routing picks one that the fallback agent can run
546        let msg = format!("{error_message}, switching to {next} for re-route");
547        if let Err(e) = store::store_set_result(
548            store,
549            repo,
550            task_id,
551            &[
552                ("agent", serde_json::json!(next)),
553                ("model", serde_json::json!("")),
554                ("last_error", serde_json::json!(msg)),
555            ],
556        )
557        .await
558        {
559            tracing::warn!(task_id, err = %e, "failed to write failover agent to store");
560        }
561
562        // Apply exponential backoff with jitter before retry.
563        // Return "new" (not "routed") so the task goes through Phase 3a routing
564        // again. This ensures the router actually re-selects a compatible agent+model
565        // via model_for_complexity() instead of bypassing routing and defaulting to
566        // claude in the dispatch loop (#1604).
567        wait_for_fallback_backoff(task_id, store, repo).await;
568
569        // Log reroute activity for successful failover
570        let details = serde_json::json!({
571            "failure_reason": error_type.type_str(),
572            "error_message": error_message,
573            "from_agent": agent_name,
574            "to_agent": next,
575            "chain": chain,
576        });
577        store::store_log_activity(
578            store,
579            repo,
580            task_id,
581            "rerouted",
582            Some("in_progress"),
583            Some("new"),
584            Some(&next),
585            None::<&str>,
586            Some(&details),
587        )
588        .await;
589
590        return FailoverOutcome::switched(next);
591    }
592
593    // No fallback available
594    tracing::warn!(task_id, agent = agent_name, "no fallback agents available");
595
596    let is_retryable = matches!(
597        error_type,
598        RetryableError::UsageLimit
599            | RetryableError::AuthError
600            | RetryableError::ModelUnavailable
601            | RetryableError::Timeout
602            | RetryableError::Interrupted
603    );
604
605    // Timeouts and interrupts are not agent faults, skip the agent cooldown
606    if !matches!(
607        error_type,
608        RetryableError::MissingTooling
609            | RetryableError::ModelUnavailable
610            | RetryableError::Timeout
611            | RetryableError::Interrupted
612    ) && !credit_recorded
613    {
614        record_agent_failure_with_message(agent_name, error_message).await;
615    }
616
617    if is_retryable {
618        // The other agents are in cooldown, so wait out the backoff and let the router
619        // pick the task up again. A fresh chain keeps the next run from seeing every agent as tried
620        wait_for_fallback_backoff(task_id, store, repo).await;
621
622        let msg = format!("{error_message} (no fallback agents, will retry)");
623        if let Err(e) = store::store_set_result(
624            store,
625            repo,
626            task_id,
627            &[
628                ("last_error", serde_json::json!(msg)),
629                ("limit_reroute_chain", serde_json::json!("")),
630            ],
631        )
632        .await
633        {
634            tracing::warn!(task_id, err = %e, "failed to write no_fallback retry fields to store");
635        }
636
637        let details = serde_json::json!({
638            "failure_reason": error_type.type_str(),
639            "error_message": error_message,
640            "chain": chain,
641            "no_fallback_available": true,
642            "is_retryable": true,
643        });
644        store::store_log_activity(
645            store,
646            repo,
647            task_id,
648            "rerouted",
649            Some("in_progress"),
650            Some("new"),
651            None::<&str>,
652            None::<&str>,
653            Some(&details),
654        )
655        .await;
656
657        return FailoverOutcome::with_status("new");
658    }
659
660    let msg = format!("{error_message}, no fallback agents");
661    if let Err(e) = store::store_set_result(
662        store,
663        repo,
664        task_id,
665        &[("last_error", serde_json::json!(msg))],
666    )
667    .await
668    {
669        tracing::warn!(task_id, err = %e, "failed to write no_fallback last_error to store");
670    }
671
672    // Log reroute activity for no fallback available
673    let details = serde_json::json!({
674        "failure_reason": error_type.type_str(),
675        "error_message": error_message,
676        "chain": chain,
677        "no_fallback_available": true,
678    });
679    store::store_log_activity(
680        store,
681        repo,
682        task_id,
683        "rerouted",
684        Some("in_progress"),
685        Some("needs_review"),
686        None::<&str>,
687        None::<&str>,
688        Some(&details),
689    )
690    .await;
691
692    FailoverOutcome::with_status("needs_review")
693}
694
695/// Count a timeout against the task. Returns "blocked" once the task has timed out
696/// `workflow.max_timeout_retries` times (default 3), so a task that always times out does not loop
697pub async fn block_after_repeated_timeouts(
698    task_id: &str,
699    error_message: &str,
700    store: &Option<Arc<TaskStore>>,
701    repo: &str,
702) -> Option<String> {
703    let max: i64 = app_config::get("workflow.max_timeout_retries")
704        .ok()
705        .and_then(|v| v.parse().ok())
706        .unwrap_or(3);
707    let count = match store::store_increment(store, repo, task_id, "timeout_retries").await {
708        Ok(count) => i64::try_from(count).unwrap_or(i64::MAX),
709        Err(e) => {
710            tracing::warn!(task_id, err = %e, "failed to increment timeout_retries");
711            return None;
712        }
713    };
714    if count < max {
715        return None;
716    }
717
718    let block_reason = format!("timed out {count} times");
719    tracing::warn!(
720        task_id,
721        count,
722        max,
723        "timeout retries exhausted, blocking task"
724    );
725    if let Err(e) = store::store_set_result(
726        store,
727        repo,
728        task_id,
729        &[
730            (
731                "last_error",
732                serde_json::json!(format!("{error_message} ({block_reason})")),
733            ),
734            ("block_reason", serde_json::json!(block_reason)),
735            ("limit_reroute_chain", serde_json::json!("")),
736        ],
737    )
738    .await
739    {
740        tracing::warn!(task_id, err = %e, "failed to write timeout block fields to store");
741    }
742    let details = serde_json::json!({
743        "failure_reason": "timeout",
744        "error_message": error_message,
745        "timeout_retries": count,
746    });
747    store::store_log_activity(
748        store,
749        repo,
750        task_id,
751        "rerouted",
752        Some("in_progress"),
753        Some("blocked"),
754        None::<&str>,
755        None::<&str>,
756        Some(&details),
757    )
758    .await;
759    Some("blocked".to_string())
760}
761
762/// Get the reroute chain from store.
763pub async fn get_reroute_chain(
764    task_id: &str,
765    store: &Option<Arc<TaskStore>>,
766    repo: &str,
767) -> String {
768    store::opt_store_get_task(store, repo, task_id)
769        .await
770        .map(|t| t.limit_reroute_chain)
771        .unwrap_or_default()
772        .trim()
773        .to_string()
774}
775
776/// Update the reroute chain in store.
777pub async fn update_reroute_chain(
778    task_id: &str,
779    current_agent: &str,
780    existing_chain: &str,
781    store: &Option<Arc<TaskStore>>,
782    repo: &str,
783) -> String {
784    let mut chain = existing_chain.to_string();
785    if chain.is_empty() {
786        chain = current_agent.to_string();
787    } else if !chain.split(',').any(|a| a == current_agent) {
788        chain = format!("{chain},{current_agent}");
789    }
790
791    if let Err(e) = store::store_set_result(
792        store,
793        repo,
794        task_id,
795        &[("limit_reroute_chain", serde_json::json!(chain))],
796    )
797    .await
798    {
799        tracing::warn!(task_id, err = %e, "failed to write limit_reroute_chain to store");
800    }
801    chain
802}
803
804/// Review response from the review agent.
805#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
806pub struct ReviewResponse {
807    /// Decision: "approve" | "request_changes"
808    pub decision: String,
809    /// Detailed review feedback.
810    pub notes: String,
811    /// Test results: "pass" | "fail" | "skipped"
812    pub test_results: Option<String>,
813    /// List of issues found.
814    pub issues: Vec<ReviewIssue>,
815}
816
817/// A single issue found during review.
818#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
819pub struct ReviewIssue {
820    /// File path.
821    pub file: String,
822    /// Line number (optional).
823    pub line: Option<u32>,
824    /// Severity: "error" | "warning"
825    pub severity: String,
826    /// Description of the issue.
827    pub description: String,
828}
829
830/// Parse a review response from already-unwrapped text.
831///
832/// Expects the raw result text (already extracted from the agent envelope
833/// by the agent-specific parser). Tries direct JSON parse, then markdown
834/// code block extraction.
835pub fn parse_review_response(text: &str) -> anyhow::Result<ReviewResponse> {
836    let resp = parse_review_json(text)?;
837    if matches!(resp.decision.as_str(), "approve" | "request_changes") {
838        return Ok(resp);
839    }
840
841    // Weak models echo the prompt template ("approve|request_changes"), strip it so it
842    // does not match the "request_changes" keyword
843    let stripped = text.replace("approve|request_changes", "");
844    infer_review_response_from_text(&stripped)
845        .ok_or_else(|| anyhow::anyhow!("invalid review decision: {}", resp.decision))
846}
847
848fn parse_review_json(text: &str) -> anyhow::Result<ReviewResponse> {
849    // Try direct JSON parse
850    if let Ok(resp) = serde_json::from_str::<ReviewResponse>(text) {
851        return Ok(resp);
852    }
853
854    // Try extracting JSON from markdown code blocks
855    if let Some(json_str) = extract_json_block(text)
856        && let Ok(resp) = serde_json::from_str::<ReviewResponse>(&json_str)
857    {
858        return Ok(resp);
859    }
860
861    if let Some(resp) = extract_review_response_object(text) {
862        return Ok(resp);
863    }
864
865    anyhow::bail!("failed to parse review response")
866}
867
868fn infer_review_response_from_text(text: &str) -> Option<ReviewResponse> {
869    let lower = text.to_ascii_lowercase();
870    let has_changes_requested =
871        lower.contains("changes requested") || lower.contains("request_changes");
872    if has_changes_requested {
873        return Some(ReviewResponse {
874            decision: "request_changes".to_string(),
875            notes: "Inferred decision from plain-text review output.".to_string(),
876            test_results: None,
877            issues: Vec::new(),
878        });
879    }
880
881    // Approval only from an explicit decision token. "approved" and CI-evidence
882    // phrases ("all checks passed") are not a decision, a negation list cannot cover them
883    let positive_approval = lower
884        .split(|c: char| !c.is_ascii_alphabetic())
885        .any(|word| word == "lgtm")
886        || lower.contains("decision is `approve`")
887        || lower.contains("decision: `approve`")
888        || lower.contains("decision is approve")
889        || lower.contains("decision: approve");
890    if positive_approval {
891        return Some(ReviewResponse {
892            decision: "approve".to_string(),
893            notes: "Inferred decision from plain-text review output.".to_string(),
894            test_results: None,
895            issues: Vec::new(),
896        });
897    }
898
899    None
900}
901
902/// Public entry point for keyword-based review response inference.
903///
904/// Used by `review.rs` when `parse_review_response` fails on the extracted text.
905pub fn infer_review_response(text: &str) -> Option<ReviewResponse> {
906    infer_review_response_from_text(text)
907}
908
909/// Extract the first valid JSON object from markdown code blocks.
910///
911/// Searches for ```json blocks and returns the first one whose content
912/// starts with `{`, skipping non-JSON code blocks.
913fn extract_json_block(text: &str) -> Option<String> {
914    let mut search_from = 0;
915    while let Some(start) = text[search_from..].find("```json") {
916        let abs_start = search_from + start;
917        let after_tag = abs_start + "```json".len();
918        let Some(rest) = text[after_tag..].strip_prefix('\n') else {
919            search_from = after_tag;
920            continue;
921        };
922        let content_start = text.len() - rest.len();
923        let Some(end_offset) = rest.find("```") else {
924            search_from = content_start;
925            continue;
926        };
927        let end = content_start + end_offset;
928        let content = text[content_start..end].trim();
929        if content.starts_with('{') {
930            return Some(content.to_string());
931        }
932        search_from = end + 3;
933    }
934    None
935}
936
937/// Extract a ReviewResponse JSON object from arbitrary text.
938///
939/// Scans for balanced JSON objects and tries to parse them as ReviewResponse.
940fn extract_review_response_object(text: &str) -> Option<ReviewResponse> {
941    let mut start: Option<usize> = None;
942    let mut depth: i32 = 0;
943    let mut in_string = false;
944    let mut escape = false;
945
946    for (idx, ch) in text.char_indices() {
947        if start.is_some() {
948            if in_string {
949                if escape {
950                    escape = false;
951                } else if ch == '\\' {
952                    escape = true;
953                } else if ch == '"' {
954                    in_string = false;
955                }
956                continue;
957            }
958
959            match ch {
960                '"' => in_string = true,
961                '{' => depth += 1,
962                '}' => {
963                    depth -= 1;
964                    if depth == 0 {
965                        debug_assert!(
966                            start.is_some(),
967                            "start must be Some when depth returns to 0"
968                        );
969                        let start_idx = start.unwrap_or(0);
970                        let candidate = &text[start_idx..=idx];
971                        if let Ok(resp) = serde_json::from_str::<ReviewResponse>(candidate) {
972                            return Some(resp);
973                        }
974                        start = None;
975                    }
976                }
977                _ => {}
978            }
979            continue;
980        }
981
982        if ch == '{' {
983            start = Some(idx);
984            depth = 1;
985            in_string = false;
986            escape = false;
987        }
988    }
989
990    None
991}
992
993/// Extract learnings and store as memory for future attempts.
994#[expect(
995    clippy::too_many_arguments,
996    reason = "a memory entry records the task, attempt, agent, model and outcome"
997)]
998pub async fn store_learnings_from_response(
999    task_id: &str,
1000    attempt: u32,
1001    agent: &str,
1002    model: Option<&str>,
1003    response: &AgentResponse,
1004    error: Option<&str>,
1005    store: &Option<Arc<TaskStore>>,
1006    repo: &str,
1007    raw_stdout: &str,
1008) {
1009    // Use agent self-reported files; fall back to NDJSON tool-call extraction when empty.
1010    let files_modified = if response.files.is_empty() {
1011        let extracted = ndjson::extract_files_changed(raw_stdout);
1012        if !extracted.is_empty() {
1013            tracing::debug!(
1014                task_id,
1015                count = extracted.len(),
1016                "files_modified: fell back to NDJSON extraction (agent reported none)"
1017            );
1018        }
1019        extracted
1020    } else {
1021        response.files.clone()
1022    };
1023
1024    // Build the memory entry
1025    let entry = MemoryEntry {
1026        attempt,
1027        agent: agent.to_string(),
1028        model: model.map(String::from),
1029        learnings: response.learnings.clone(),
1030        error: error.map(String::from),
1031        files_modified,
1032        approach: response.summary.clone(),
1033        timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1034    };
1035
1036    if let Some(st) = store
1037        && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1038    {
1039        match st.append_memory(store_id, &entry).await {
1040            Err(e) => {
1041                tracing::warn!(task_id, error = ?e, "failed to store memory");
1042            }
1043            _ => {
1044                tracing::debug!(task_id, attempt, "stored memory for attempt");
1045            }
1046        }
1047    }
1048}
1049
1050/// Store a memory entry for a failed attempt (without a full AgentResponse).
1051pub async fn store_failure_memory(
1052    task_id: &str,
1053    attempt: u32,
1054    agent: &str,
1055    model: Option<&str>,
1056    error: &str,
1057    store: &Option<Arc<TaskStore>>,
1058    repo: &str,
1059) {
1060    let entry = MemoryEntry {
1061        attempt,
1062        agent: agent.to_string(),
1063        model: model.map(String::from),
1064        learnings: vec![],
1065        error: Some(error.to_string()),
1066        files_modified: vec![],
1067        approach: String::new(),
1068        timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1069    };
1070
1071    if let Some(st) = store
1072        && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1073    {
1074        match st.append_memory(store_id, &entry).await {
1075            Err(e) => {
1076                tracing::warn!(task_id, error = ?e, "failed to store failure memory");
1077            }
1078            _ => {
1079                tracing::debug!(task_id, attempt, "stored failure memory");
1080            }
1081        }
1082    }
1083}
1084
1085#[cfg(test)]
1086mod tests {
1087    use std::env;
1088    #[cfg(unix)]
1089    use std::os::unix::fs::PermissionsExt;
1090    use std::process;
1091    use std::sync::Arc;
1092
1093    use super::super::TaskRunner;
1094    use super::*;
1095    use crate::backends::{ExternalId, ExternalTask};
1096    use crate::engine::cooldown;
1097    use crate::engine::router::{Router, RouterConfig};
1098    use crate::engine::runner::agents::{AgentError, find_agent_result, patterns};
1099    use crate::store::TaskStatus;
1100    use serial_test::serial;
1101
1102    #[serial(cooldown_state)]
1103    #[test]
1104    fn calculate_backoff_delay_jitter_is_centered() {
1105        // Run many samples and verify the range is [0.7*capped, 1.3*capped].
1106        // We can't control `now` directly, but calling the function many times
1107        // exercises different microsecond values and lets us check bounds.
1108        let base = config::retry_base_delay_ms();
1109        let max = config::retry_max_delay_ms();
1110
1111        // retry_count=0 → exponential = base, capped = base (assuming base < max)
1112        let capped = base.min(max);
1113        #[expect(
1114            clippy::cast_precision_loss,
1115            clippy::cast_possible_truncation,
1116            clippy::cast_sign_loss,
1117            reason = "mirrors the jitter range in calculate_backoff_delay"
1118        )]
1119        let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
1120        let lo = capped.saturating_sub(jitter_range);
1121        let hi = capped + jitter_range;
1122
1123        for _ in 0..200 {
1124            let delay = calculate_backoff_delay(0);
1125            assert!(
1126                delay >= lo && delay <= hi,
1127                "delay {delay} out of expected range [{lo}, {hi}]"
1128            );
1129        }
1130    }
1131
1132    #[serial(cooldown_state)]
1133    #[test]
1134    fn fallback_retry_count_uses_larger_network_retry_streak() {
1135        let task = Task {
1136            id: 1,
1137            external_id: None,
1138            repo: "owner/repo".to_string(),
1139            origin: "internal".to_string(),
1140            title: "test".to_string(),
1141            body: String::new(),
1142            status: TaskStatus::New,
1143            source: String::new(),
1144            source_id: String::new(),
1145            author: String::new(),
1146            url: String::new(),
1147            labels: vec![],
1148            agent: None,
1149            model: None,
1150            complexity: "simple".to_string(),
1151            estimate: 0,
1152            route_reason: String::new(),
1153            agent_profile: String::new(),
1154            route_attempts: 0,
1155            attempts: 0,
1156            branch: String::new(),
1157            worktree: String::new(),
1158            worktree_cleaned: false,
1159            owner_node: None,
1160            summary: String::new(),
1161            last_error: String::new(),
1162            parent_id: None,
1163            block_reason: None,
1164            pr_number: None,
1165            pr_review_context: String::new(),
1166            last_review_ts: String::new(),
1167            review_ts_map: "{}".to_string(),
1168            last_comment_review_ts: String::new(),
1169            merge_conflict_retries: 0,
1170            ci_merge_failures: 0,
1171            pr_create_failures: 0,
1172            push_failures: 0,
1173            network_retries: 3,
1174            timeout_retries: 0,
1175            review_agent_failures: 0,
1176            review_resets: 0,
1177            review_cycles: 0,
1178            review_invocations: 0,
1179            needs_review_refires: 0,
1180            input_tokens: 0,
1181            output_tokens: 0,
1182            total_cost_usd: 0.0,
1183            model_reroute_chain: String::new(),
1184            limit_reroute_chain: "claude".to_string(),
1185            memory: vec![],
1186            delegations: vec![],
1187            auto_unblock_count: 0,
1188            auto_unblock_last_at: String::new(),
1189            auto_unblock_last_reason: String::new(),
1190            ci_recovery_count: 0,
1191            no_code_reroutes: 0,
1192            no_code_last_agent: String::new(),
1193            created_at: String::new(),
1194            updated_at: String::new(),
1195        };
1196
1197        assert_eq!(fallback_retry_count(Some(&task)), 3);
1198    }
1199
1200    #[serial(cooldown_state)]
1201    #[test]
1202    fn retryable_type_str_returns_correct_labels() {
1203        assert_eq!(RetryableError::Timeout.type_str(), "timeout");
1204        assert_eq!(RetryableError::Interrupted.type_str(), "interrupted");
1205        assert_eq!(RetryableError::UsageLimit.type_str(), "rate_limit");
1206        assert_eq!(RetryableError::AuthError.type_str(), "auth_error");
1207        assert_eq!(RetryableError::Failed.type_str(), "failed");
1208        assert_eq!(RetryableError::MissingTooling.type_str(), "failed");
1209    }
1210
1211    #[serial(cooldown_state)]
1212    #[test]
1213    fn is_usage_limit_detects_rate_limit() {
1214        assert!(patterns::detect_rate_limit("Error: rate limit exceeded").is_some());
1215        assert!(patterns::detect_rate_limit("HTTP 429 Too Many Requests").is_some());
1216        assert!(patterns::detect_rate_limit("quota exceeded for model").is_some());
1217        // context_length_exceeded is handled by detect_context_overflow, not detect_rate_limit
1218        assert!(patterns::detect_context_overflow("context_length_exceeded").is_some());
1219    }
1220
1221    #[serial(cooldown_state)]
1222    #[test]
1223    fn is_usage_limit_rejects_normal_text() {
1224        assert!(patterns::detect_rate_limit("task completed successfully").is_none());
1225        assert!(patterns::detect_rate_limit("").is_none());
1226    }
1227
1228    #[serial(cooldown_state)]
1229    #[test]
1230    fn is_auth_error_detects_common_patterns() {
1231        assert!(patterns::detect_auth_error("401 Unauthorized").is_some());
1232        assert!(patterns::detect_auth_error("invalid api key provided").is_some());
1233        assert!(patterns::detect_auth_error("Your billing plan has expired").is_some());
1234        assert!(patterns::detect_auth_error("Error: 403 Forbidden").is_some());
1235    }
1236
1237    #[serial(cooldown_state)]
1238    #[test]
1239    fn is_auth_error_rejects_normal_text() {
1240        assert!(patterns::detect_auth_error("task completed successfully").is_none());
1241        assert!(patterns::detect_auth_error("").is_none());
1242    }
1243
1244    #[serial(cooldown_state)]
1245    #[test]
1246    fn detect_missing_tooling_finds_known_tools() {
1247        let result = patterns::detect_missing_tool("bun: command not found");
1248        assert!(result.is_some());
1249        if let AgentError::MissingTool { tool } = result.unwrap() {
1250            assert_eq!(tool, "bun");
1251        } else {
1252            panic!("expected MissingTool error");
1253        }
1254
1255        let result = patterns::detect_missing_tool("env: anchor: no such file");
1256        assert!(result.is_some());
1257        if let AgentError::MissingTool { tool } = result.unwrap() {
1258            assert_eq!(tool, "anchor");
1259        } else {
1260            panic!("expected MissingTool error");
1261        }
1262
1263        let result = patterns::detect_missing_tool("spawn docker enoent");
1264        assert!(result.is_some());
1265        if let AgentError::MissingTool { tool } = result.unwrap() {
1266            assert_eq!(tool, "docker");
1267        } else {
1268            panic!("expected MissingTool error");
1269        }
1270    }
1271
1272    #[serial(cooldown_state)]
1273    #[test]
1274    fn detect_missing_tooling_returns_none_for_normal() {
1275        assert!(patterns::detect_missing_tool("everything works fine").is_none());
1276        assert!(patterns::detect_missing_tool("").is_none());
1277    }
1278
1279    // ── Cooldown tests ────────────────────────────────────────────
1280
1281    #[serial(cooldown_state)]
1282    #[tokio::test]
1283    async fn record_and_check_agent_cooldown() {
1284        cooldown::reset_global_state().await;
1285        // Use unique names to avoid interference from other tests
1286        let agent = "test_cooldown_agent_1";
1287        assert!(!is_agent_in_cooldown(agent));
1288        record_agent_failure_with_message(agent, "").await;
1289        assert!(is_agent_in_cooldown(agent));
1290    }
1291
1292    #[serial(cooldown_state)]
1293    #[tokio::test]
1294    async fn record_and_check_model_cooldown() {
1295        cooldown::reset_global_state().await;
1296        let agent = "test_cooldown_agent_2";
1297        let model = "test_model_x";
1298        assert!(!is_model_in_cooldown(agent, model));
1299        record_model_failure(agent, model).await;
1300        assert!(is_model_in_cooldown(agent, model));
1301        // Different model should not be in cooldown
1302        assert!(!is_model_in_cooldown(agent, "other_model"));
1303    }
1304
1305    #[serial(cooldown_state)]
1306    #[tokio::test]
1307    async fn clear_expired_does_not_remove_fresh_entries() {
1308        cooldown::reset_global_state().await;
1309        let agent = "test_cooldown_agent_3";
1310        record_agent_failure_with_message(agent, "").await;
1311        clear_expired_cooldowns();
1312        // Should still be in cooldown (just recorded)
1313        assert!(is_agent_in_cooldown(agent));
1314    }
1315
1316    // Use fake agent names so other tests don't interfere.
1317    #[serial(cooldown_state)]
1318    #[test]
1319    fn pick_fallback_skips_current_agent() {
1320        let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1321        let result = pick_fallback_agent("test_agent_a", "", &available);
1322        assert_eq!(result, Some("test_agent_b".to_string()));
1323    }
1324
1325    #[serial(cooldown_state)]
1326    #[test]
1327    fn pick_fallback_skips_chain_agents() {
1328        let available = vec![
1329            "test_agent_a".to_string(),
1330            "test_agent_b".to_string(),
1331            "test_agent_c".to_string(),
1332        ];
1333        let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1334        assert_eq!(result, Some("test_agent_c".to_string()));
1335    }
1336
1337    #[serial(cooldown_state)]
1338    #[test]
1339    fn pick_fallback_returns_none_when_exhausted() {
1340        let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1341        let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1342        assert!(result.is_none());
1343    }
1344
1345    #[serial(cooldown_state)]
1346    #[test]
1347    fn weight_signal_variants() {
1348        let success = WeightSignal::Success {
1349            agent: "claude".to_string(),
1350        };
1351        let limited = WeightSignal::RateLimited {
1352            agent: "codex".to_string(),
1353        };
1354        let none = WeightSignal::None;
1355
1356        // Verify Debug trait
1357        assert!(format!("{success:?}").contains("claude"));
1358        assert!(format!("{limited:?}").contains("codex"));
1359        assert!(format!("{none:?}").contains("None"));
1360    }
1361
1362    #[serial(cooldown_state)]
1363    #[test]
1364    fn parse_review_response_direct_json() {
1365        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1366        let resp = parse_review_response(json).unwrap();
1367        assert_eq!(resp.decision, "approve");
1368        assert_eq!(resp.notes, "LGTM");
1369        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1370    }
1371
1372    #[serial(cooldown_state)]
1373    #[test]
1374    fn parse_review_response_from_markdown() {
1375        let md = r#"Here is my review:
1376
1377```json
1378{"decision":"request_changes","notes":"Fix the bug","issues":[{"file":"src/main.rs","line":10,"severity":"error","description":"null deref"}]}
1379```
1380
1381That's all."#;
1382        let resp = parse_review_response(md).unwrap();
1383        assert_eq!(resp.decision, "request_changes");
1384        assert_eq!(resp.issues.len(), 1);
1385        assert_eq!(resp.issues[0].file, "src/main.rs");
1386    }
1387
1388    #[serial(cooldown_state)]
1389    #[test]
1390    fn parse_review_response_embedded_json_object() {
1391        let text = "Review complete.\n\n{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}\nThanks!";
1392        let resp = parse_review_response(text).unwrap();
1393        assert_eq!(resp.decision, "approve");
1394        assert_eq!(resp.notes, "LGTM");
1395    }
1396
1397    #[serial(cooldown_state)]
1398    #[test]
1399    fn parse_review_response_template_echo_infers_from_notes() {
1400        let json = r#"{"decision":"approve|request_changes","notes":"LGTM","issues":[]}"#;
1401        assert_eq!(parse_review_response(json).unwrap().decision, "approve");
1402
1403        let no_signal = r#"{"decision":"approve|request_changes","notes":"hmm","issues":[]}"#;
1404        assert!(parse_review_response(no_signal).is_err());
1405    }
1406
1407    #[serial(cooldown_state)]
1408    #[test]
1409    fn parse_review_response_invalid() {
1410        let result = parse_review_response("not json at all");
1411        assert!(result.is_err());
1412    }
1413
1414    #[serial(cooldown_state)]
1415    #[test]
1416    fn extract_json_block_basic() {
1417        let md = "text\n```json\n{\"key\": \"value\"}\n```\nmore";
1418        let result = extract_json_block(md);
1419        assert!(result.is_some());
1420        assert!(result.unwrap().contains("\"key\""));
1421    }
1422
1423    #[serial(cooldown_state)]
1424    #[test]
1425    fn extract_json_block_skips_non_json_blocks() {
1426        let md = "```json\nnot-json-array\n```\n\n```json\n{\"real\": true}\n```";
1427        let result = extract_json_block(md);
1428        assert!(result.is_some());
1429        assert!(result.unwrap().contains("\"real\""));
1430    }
1431
1432    #[serial(cooldown_state)]
1433    #[test]
1434    fn extract_json_block_none_when_missing() {
1435        assert!(extract_json_block("no code blocks here").is_none());
1436    }
1437
1438    #[serial(cooldown_state)]
1439    #[test]
1440    fn extract_json_block_skips_malformed_intermediate_blocks() {
1441        let md = "prefix ```json{\"broken\": true}\n\n```json\n{\"real\": true}\n```";
1442        let result = extract_json_block(md);
1443        assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1444    }
1445
1446    #[serial(cooldown_state)]
1447    #[test]
1448    fn extract_json_block_skips_multiple_malformed_intermediate_blocks() {
1449        let md = "prefix ```json{\"broken\": true}\nmid ```json[1, 2, 3]\n\n```json\n{\"real\": true}\n```";
1450        let result = extract_json_block(md);
1451        assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1452    }
1453
1454    // ── parse_review_response + infer_review_response (production path) ─
1455
1456    /// Helper that mirrors the production plain-text path in review.rs:
1457    /// parse_review_response → infer_review_response.
1458    fn parse_review_plain(text: &str) -> anyhow::Result<ReviewResponse> {
1459        parse_review_response(text).or_else(|_| {
1460            infer_review_response(text)
1461                .ok_or_else(|| anyhow::anyhow!("failed to parse review response"))
1462        })
1463    }
1464
1465    #[serial(cooldown_state)]
1466    #[test]
1467    fn parse_review_from_output_direct_json() {
1468        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1469        let resp = parse_review_response(json).unwrap();
1470        assert_eq!(resp.decision, "approve");
1471    }
1472
1473    #[serial(cooldown_state)]
1474    #[test]
1475    fn parse_review_from_output_markdown() {
1476        let md = "Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}\n```\n";
1477        let resp = parse_review_response(md).unwrap();
1478        assert_eq!(resp.decision, "request_changes");
1479    }
1480
1481    #[serial(cooldown_state)]
1482    #[test]
1483    fn parse_review_from_output_plain_text_fallback() {
1484        let text = "The review is done. LGTM.";
1485        let resp = parse_review_plain(text).unwrap();
1486        assert_eq!(resp.decision, "approve");
1487        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1488    }
1489
1490    /// opencode NDJSON stream — text events use the `part.text` format.
1491    #[serial(cooldown_state)]
1492    #[test]
1493    fn parse_review_from_output_ndjson_part_format() {
1494        let ndjson = concat!(
1495            r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1496            "\n",
1497            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Reviewing the PR...\n\n```json\n{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1498            "\n",
1499            r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop"}}"#,
1500        );
1501        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1502        assert_eq!(resp.decision, "approve");
1503        assert_eq!(resp.notes, "All checks pass");
1504    }
1505
1506    /// opencode NDJSON stream — text events use the newer `text` directly-in-event format.
1507    #[serial(cooldown_state)]
1508    #[test]
1509    fn parse_review_from_output_ndjson_direct_text_format() {
1510        let ndjson = concat!(
1511            r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1512            "\n",
1513            r#"{"type":"text","timestamp":1001,"sessionID":"ses_abc","text":"Here is my review:\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the bug\",\"test_results\":\"fail\",\"issues\":[]}\n```\n"}"#,
1514            "\n",
1515            r#"{"type":"step_finish","timestamp":1002,"sessionID":"ses_abc"}"#,
1516        );
1517        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1518        assert_eq!(resp.decision, "request_changes");
1519        assert_eq!(resp.notes, "Fix the bug");
1520    }
1521
1522    /// opencode NDJSON with ReviewResponse JSON directly in a text event (no markdown wrapper).
1523    #[serial(cooldown_state)]
1524    #[test]
1525    fn parse_review_from_output_ndjson_direct_json_in_text_event() {
1526        let ndjson = concat!(
1527            r#"{"type":"step_start","timestamp":1000}"#,
1528            "\n",
1529            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1530            "\n",
1531            r#"{"type":"step_finish","timestamp":1002}"#,
1532        );
1533        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1534        assert_eq!(resp.decision, "approve");
1535    }
1536
1537    #[serial(cooldown_state)]
1538    #[test]
1539    fn parse_review_from_output_ndjson_prefers_final_json_text_event() {
1540        let ndjson = concat!(
1541            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Working on it..."}}"#,
1542            "\n",
1543            r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1544        );
1545        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1546        assert_eq!(resp.decision, "approve");
1547        assert_eq!(resp.notes, "LGTM");
1548    }
1549
1550    /// Concatenated text events produce a valid ReviewResponse.
1551    #[serial(cooldown_state)]
1552    #[test]
1553    fn parse_review_from_output_ndjson_concatenated_text() {
1554        let ndjson = concat!(
1555            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Here is my\n"}}"#,
1556            "\n",
1557            r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"review:\n\n```json\n{\"decision\":\"approve\",\"notes\":\"OK\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1558        );
1559        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1560        assert_eq!(resp.decision, "approve");
1561    }
1562
1563    /// codex NDJSON stream — agent_message item carries the ReviewResponse JSON.
1564    ///
1565    /// Real format from codex `exec --json` output (observed 2026-03-06):
1566    /// thread.started / turn.started / item.completed(reasoning) / item.completed(agent_message) / turn.completed
1567    #[serial(cooldown_state)]
1568    #[test]
1569    fn parse_review_from_output_codex_agent_message() {
1570        let ndjson = concat!(
1571            r#"{"type":"thread.started","thread_id":"t1"}"#,
1572            "\n",
1573            r#"{"type":"turn.started"}"#,
1574            "\n",
1575            r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Planning git commands..."}}"#,
1576            "\n",
1577            r#"{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"git log --oneline -5","output":"abc123 fix: something"}}"#,
1578            "\n",
1579            r#"{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1580            "\n",
1581            r#"{"type":"turn.completed"}"#,
1582        );
1583        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1584        assert_eq!(resp.decision, "approve");
1585        assert_eq!(resp.notes, "All checks pass");
1586    }
1587
1588    /// codex NDJSON with ReviewResponse in a markdown code block inside agent_message.
1589    #[serial(cooldown_state)]
1590    #[test]
1591    fn parse_review_from_output_codex_agent_message_markdown() {
1592        let ndjson = concat!(
1593            r#"{"type":"thread.started","thread_id":"t1"}"#,
1594            "\n",
1595            r#"{"type":"turn.started"}"#,
1596            "\n",
1597            r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Reviewing..."}}"#,
1598            "\n",
1599            r#"{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the tests\",\"issues\":[]}\n```\n"}}"#,
1600            "\n",
1601            r#"{"type":"turn.completed"}"#,
1602        );
1603        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1604        assert_eq!(resp.decision, "request_changes");
1605        assert_eq!(resp.notes, "Fix the tests");
1606    }
1607
1608    /// codex reasoning items must NOT be extracted as review text — only agent_message is used.
1609    #[serial(cooldown_state)]
1610    #[test]
1611    fn codex_skips_reasoning_items() {
1612        let ndjson = concat!(
1613            r#"{"type":"item.completed","item":{"type":"reasoning","text":"internal thoughts..."}}"#,
1614            "\n",
1615            r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}}"#,
1616        );
1617        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1618        assert_eq!(resp.decision, "approve");
1619        assert_eq!(resp.notes, "LGTM");
1620    }
1621
1622    /// Negation patterns must NOT trigger a false approval.
1623    #[serial(cooldown_state)]
1624    #[test]
1625    fn infer_review_response_negation_not_approved() {
1626        assert!(
1627            parse_review_plain("The changes were not approved.").is_err(),
1628            "\"not approved\" should not infer approval"
1629        );
1630    }
1631
1632    #[serial(cooldown_state)]
1633    #[test]
1634    fn infer_review_response_negation_unapproved() {
1635        assert!(
1636            parse_review_plain("Unapproved changes found in the diff.").is_err(),
1637            "\"unapproved\" should not infer approval"
1638        );
1639    }
1640
1641    #[serial(cooldown_state)]
1642    #[test]
1643    fn infer_review_response_negation_not_approving() {
1644        assert!(
1645            parse_review_plain("I am not approving this PR.").is_err(),
1646            "\"not approving\" should not infer approval"
1647        );
1648    }
1649
1650    #[serial(cooldown_state)]
1651    #[test]
1652    fn infer_review_response_negation_cannot_be_approved() {
1653        assert!(
1654            parse_review_plain("This PR cannot be approved until the race in tick.rs is fixed.")
1655                .is_err(),
1656            "\"cannot be approved\" should not infer approval"
1657        );
1658    }
1659
1660    #[serial(cooldown_state)]
1661    #[test]
1662    fn infer_review_response_negation_should_not_be_approved() {
1663        assert!(
1664            parse_review_plain("The diff breaks the build and should not be approved.").is_err(),
1665            "\"should not be approved\" should not infer approval"
1666        );
1667    }
1668
1669    #[serial(cooldown_state)]
1670    #[test]
1671    fn infer_review_response_negation_not_yet_approved() {
1672        assert!(
1673            parse_review_plain("This change is not yet approved.").is_err(),
1674            "\"not yet approved\" should not infer approval"
1675        );
1676    }
1677
1678    #[serial(cooldown_state)]
1679    #[test]
1680    fn infer_review_response_negation_wont_approve() {
1681        assert!(
1682            parse_review_plain("I won't approve this until tests cover the retry path.").is_err(),
1683            "\"won't approve\" should not infer approval"
1684        );
1685    }
1686
1687    #[serial(cooldown_state)]
1688    #[test]
1689    fn infer_review_response_bare_approved() {
1690        assert!(
1691            parse_review_plain("The PR was approved.").is_err(),
1692            "bare \"approved\" should not infer approval"
1693        );
1694    }
1695
1696    /// Positive plain-text approval signals must still work.
1697    #[serial(cooldown_state)]
1698    #[test]
1699    fn infer_review_response_lgtm() {
1700        let resp = parse_review_plain("LGTM, everything looks fine.").unwrap();
1701        assert_eq!(resp.decision, "approve");
1702    }
1703
1704    /// Evidence phrases are not a decision token
1705    #[serial(cooldown_state)]
1706    #[test]
1707    fn infer_review_response_looks_good() {
1708        assert!(
1709            parse_review_plain("Looks good to me!").is_err(),
1710            "\"looks good\" should not infer approval"
1711        );
1712    }
1713
1714    #[serial(cooldown_state)]
1715    #[test]
1716    fn infer_review_response_all_checks_passed() {
1717        assert!(
1718            parse_review_plain(
1719                "The background task completed but I already got the CI results directly. All checks passed — no action needed.",
1720            )
1721            .is_err(),
1722            "\"all checks passed\" should not infer approval"
1723        );
1724    }
1725
1726    #[serial(cooldown_state)]
1727    #[test]
1728    fn infer_review_response_checks_passed() {
1729        assert!(
1730            parse_review_plain("All checks passed (fmt ✓, clippy ✓, tests ✓).").is_err(),
1731            "\"all checks passed\" should not infer approval"
1732        );
1733    }
1734
1735    #[serial(cooldown_state)]
1736    #[test]
1737    fn infer_review_response_all_tests_passed() {
1738        assert!(
1739            parse_review_plain("All tests passed and the implementation looks correct.").is_err(),
1740            "\"all tests passed\" should not infer approval"
1741        );
1742    }
1743
1744    #[serial(cooldown_state)]
1745    #[test]
1746    fn infer_review_response_no_issues_found() {
1747        assert!(
1748            parse_review_plain("Review complete. No issues found in the diff.").is_err(),
1749            "\"no issues found\" should not infer approval"
1750        );
1751    }
1752
1753    /// Partial "tests passed" mid-sentence must NOT infer approval without "all" prefix.
1754    #[serial(cooldown_state)]
1755    #[test]
1756    fn infer_review_response_partial_tests_passed_no_approve() {
1757        let text = "The existing tests passed before this change but the new payment logic has a null-pointer risk on line 42.";
1758        assert!(parse_review_plain(text).is_err());
1759    }
1760
1761    /// Partial "checks passed" mid-sentence must NOT infer approval without "all" prefix.
1762    #[serial(cooldown_state)]
1763    #[test]
1764    fn infer_review_response_partial_checks_passed_no_approve() {
1765        let text = "CI checks passed for the base branch but this PR adds a broken path.";
1766        assert!(parse_review_plain(text).is_err());
1767    }
1768
1769    /// LLMs sometimes echo the JSON decision field value literally: "Decision is `approve`."
1770    #[serial(cooldown_state)]
1771    #[test]
1772    fn infer_review_response_decision_is_approve_backtick() {
1773        let text = "Already retrieved the output and completed the review above. The CI test check passed (3m11s) and all other checks are green. Decision is `approve`.";
1774        let resp = parse_review_plain(text).unwrap();
1775        assert_eq!(resp.decision, "approve");
1776    }
1777
1778    /// "Decision: approve" without backticks should also match.
1779    #[serial(cooldown_state)]
1780    #[test]
1781    fn infer_review_response_decision_colon_approve() {
1782        let text = "Reviewed the PR. No blocking issues found. Decision: approve";
1783        let resp = parse_review_plain(text).unwrap();
1784        assert_eq!(resp.decision, "approve");
1785    }
1786
1787    /// Bare backtick `approve` in a negative context must NOT infer approval.
1788    /// e.g. "do not `approve`" previously matched the removed `lower.contains("`approve`")` check.
1789    #[serial(cooldown_state)]
1790    #[test]
1791    fn infer_review_response_negation_do_not_approve_backtick() {
1792        let text = "There are unresolved issues — do not `approve` this PR until they are fixed.";
1793        assert!(
1794            parse_review_plain(text).is_err(),
1795            "negative context with backtick approve must not infer approval"
1796        );
1797    }
1798
1799    // ── review parsing via find_agent_result + parse_review_response ────
1800
1801    /// Helper that mirrors the production path in review.rs:
1802    /// find_agent_result → parse_review_response → infer_review_response.
1803    fn parse_via_agent_path(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
1804        let text = find_agent_result(agent, output)
1805            .map(|r| r.result_text)
1806            .filter(|t| !t.is_empty())
1807            .unwrap_or_else(|| output.to_string());
1808        if let Ok(r) = parse_review_response(&text) {
1809            return Ok(r);
1810        }
1811        infer_review_response_from_text(&text)
1812            .ok_or_else(|| anyhow::anyhow!("failed to parse review response from {agent} output"))
1813    }
1814
1815    #[serial(cooldown_state)]
1816    #[test]
1817    fn agent_output_claude_ndjson_review() {
1818        let ndjson = concat!(
1819            r#"{"type":"system","subtype":"init","session_id":"s1"}"#,
1820            "\n",
1821            r#"{"type":"assistant","message":{"content":[{"type":"text","text":"Reviewing..."}]}}"#,
1822            "\n",
1823            r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}","usage":{"input_tokens":100,"output_tokens":20}}"#,
1824        );
1825        let resp = parse_via_agent_path("claude", ndjson).unwrap();
1826        assert_eq!(resp.decision, "approve");
1827    }
1828
1829    #[serial(cooldown_state)]
1830    #[test]
1831    fn agent_output_opencode_ndjson_review() {
1832        let ndjson = concat!(
1833            r#"{"type":"step_start","timestamp":1000}"#,
1834            "\n",
1835            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}"}}"#,
1836            "\n",
1837            r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop","tokens":{"input":100,"output":10}}}"#,
1838        );
1839        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1840        assert_eq!(resp.decision, "request_changes");
1841    }
1842
1843    #[serial(cooldown_state)]
1844    #[test]
1845    fn agent_output_codex_ndjson_review() {
1846        let ndjson = concat!(
1847            r#"{"type":"thread.started","thread_id":"t1"}"#,
1848            "\n",
1849            r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All good\",\"issues\":[]}"}}"#,
1850            "\n",
1851            r#"{"type":"turn.completed"}"#,
1852        );
1853        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1854        assert_eq!(resp.decision, "approve");
1855    }
1856
1857    #[serial(cooldown_state)]
1858    #[test]
1859    fn agent_output_plain_text_keyword_fallback() {
1860        let text = "All checks passed, LGTM.";
1861        let resp = parse_via_agent_path("claude", text).unwrap();
1862        assert_eq!(resp.decision, "approve");
1863    }
1864
1865    #[serial(cooldown_state)]
1866    #[test]
1867    fn agent_output_direct_json() {
1868        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1869        let resp = parse_via_agent_path("opencode", json).unwrap();
1870        assert_eq!(resp.decision, "approve");
1871    }
1872
1873    /// Regression: plain text containing JSON fragments should NOT trigger false
1874    /// positives. The per-agent extractor returns None for non-NDJSON text,
1875    /// so we fall through to keyword inference instead of generic NDJSON parsing.
1876    #[serial(cooldown_state)]
1877    #[test]
1878    fn agent_output_text_with_json_fragment_no_false_positive() {
1879        let text = r#"I checked the file and found {"type":"text"} in the output. No issues."#;
1880        // This should NOT extract a review from the JSON fragment.
1881        // With the generic parser, this would match as a "text" NDJSON event.
1882        // With per-agent extraction, it returns None (not valid agent NDJSON).
1883        let result = parse_via_agent_path("claude", text);
1884        // Should either fail or infer from keywords, NOT parse the JSON fragment
1885        assert!(
1886            result.is_err() || result.as_ref().is_ok_and(|r| r.decision != "text"),
1887            "should not extract review from embedded JSON fragment"
1888        );
1889    }
1890
1891    /// Prose containing the word "decision" and a JSON-like fragment must not
1892    /// be misidentified as a review response by `extract_review_response_object`.
1893    #[serial(cooldown_state)]
1894    #[test]
1895    fn parse_review_response_ignores_prose_with_decision_word() {
1896        let text = r#"The decision was made. Here is a debug trace: {"trace":123}"#;
1897        assert!(
1898            parse_review_response(text).is_err(),
1899            "prose with 'decision' word and unrelated JSON should not parse as ReviewResponse"
1900        );
1901    }
1902
1903    /// Plain text with an embedded JSON object that has unrelated keys should
1904    /// not be extracted as a review response.
1905    #[serial(cooldown_state)]
1906    #[test]
1907    fn parse_review_response_ignores_embedded_json_without_decision_key() {
1908        let text = r#"Analysis complete. Config: {"timeout":30,"retries":3}. Done."#;
1909        assert!(
1910            parse_review_response(text).is_err(),
1911            "embedded JSON without 'decision' key should not parse as ReviewResponse"
1912        );
1913    }
1914
1915    // ── read_file_with_limit tests ───────────────────────────────────────────
1916
1917    #[serial(cooldown_state)]
1918    #[test]
1919    fn read_file_with_limit_reads_small_file_completely() {
1920        let temp_dir = env::temp_dir();
1921        let file_path = temp_dir.join(format!("orch_test_small_{}.txt", process::id()));
1922        let content = "Hello, World!";
1923        fs::write(&file_path, content).unwrap();
1924
1925        let result = read_file_with_limit(&file_path, 1024);
1926        assert!(result.is_ok());
1927        assert_eq!(result.unwrap(), content);
1928
1929        let _ = fs::remove_file(&file_path);
1930    }
1931
1932    #[serial(cooldown_state)]
1933    #[test]
1934    fn read_file_with_limit_truncates_large_file() {
1935        let temp_dir = env::temp_dir();
1936        let file_path = temp_dir.join(format!("orch_test_large_{}.txt", process::id()));
1937
1938        // Create a file larger than the limit (100 bytes)
1939        let large_content = "x".repeat(200);
1940        fs::write(&file_path, &large_content).unwrap();
1941
1942        // Read with a 50-byte limit
1943        let result = read_file_with_limit(&file_path, 50);
1944        assert!(result.is_ok());
1945
1946        let read_content = result.unwrap();
1947        // Should only read 50 bytes (the tail of the file)
1948        assert_eq!(read_content.len(), 50);
1949        assert_eq!(read_content, "x".repeat(50));
1950
1951        let _ = fs::remove_file(&file_path);
1952    }
1953
1954    #[serial(cooldown_state)]
1955    #[test]
1956    fn read_file_with_limit_handles_empty_file() {
1957        let temp_dir = env::temp_dir();
1958        let file_path = temp_dir.join(format!("orch_test_empty_{}.txt", process::id()));
1959        fs::write(&file_path, "").unwrap();
1960
1961        let result = read_file_with_limit(&file_path, 1024);
1962        assert!(result.is_ok());
1963        assert_eq!(result.unwrap(), "");
1964
1965        let _ = fs::remove_file(&file_path);
1966    }
1967
1968    #[serial(cooldown_state)]
1969    #[test]
1970    fn read_file_with_limit_handles_nonexistent_file() {
1971        let temp_dir = env::temp_dir();
1972        let file_path = temp_dir.join("orch_test_nonexistent_xyz123.txt");
1973
1974        let result = read_file_with_limit(&file_path, 1024);
1975        assert!(result.is_err());
1976    }
1977
1978    #[serial(cooldown_state)]
1979    #[test]
1980    fn read_file_with_limit_reads_exact_limit() {
1981        let temp_dir = env::temp_dir();
1982        let file_path = temp_dir.join(format!("orch_test_exact_{}.txt", process::id()));
1983
1984        // Create exactly 100 bytes
1985        let content = "y".repeat(100);
1986        fs::write(&file_path, &content).unwrap();
1987
1988        // Read with exactly 100-byte limit (should read all)
1989        let result = read_file_with_limit(&file_path, 100);
1990        assert!(result.is_ok());
1991        assert_eq!(result.unwrap().len(), 100);
1992
1993        let _ = fs::remove_file(&file_path);
1994    }
1995
1996    #[serial(cooldown_state)]
1997    #[test]
1998    fn read_file_with_limit_handles_unicode_truncation() {
1999        let temp_dir = env::temp_dir();
2000        let file_path = temp_dir.join(format!("orch_test_unicode_{}.txt", process::id()));
2001
2002        // Create content with multi-byte UTF-8 characters
2003        let emoji = "🎉".repeat(100); // Each emoji is 4 bytes
2004        fs::write(&file_path, &emoji).unwrap();
2005
2006        // Read with a limit that cuts into the emoji sequence
2007        let result = read_file_with_limit(&file_path, 50);
2008        assert!(result.is_ok());
2009
2010        let read_content = result.unwrap();
2011        // from_utf8_lossy should handle invalid sequences
2012        assert_ne!(read_content, "");
2013
2014        let _ = fs::remove_file(&file_path);
2015    }
2016
2017    // ── Fixture-based review integration tests ──────────────────────────────────
2018
2019    /// Full review pipeline: find_agent_result → parse_review_response → infer_review_response.
2020    fn parse_review_via_agent(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
2021        let text = find_agent_result(agent, output)
2022            .map(|r| r.result_text)
2023            .filter(|t| !t.is_empty())
2024            .unwrap_or_else(|| output.to_string());
2025        if let Ok(r) = parse_review_response(&text) {
2026            return Ok(r);
2027        }
2028        infer_review_response(&text)
2029            .ok_or_else(|| anyhow::anyhow!("failed to parse review from {agent} output"))
2030    }
2031
2032    // ── Claude review fixtures ─────────────────────────────────────────────────
2033
2034    #[serial(cooldown_state)]
2035    #[test]
2036    fn fixture_claude_review_approve() {
2037        let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2038        let resp = parse_review_via_agent("claude", raw).unwrap();
2039        assert_eq!(resp.decision, "approve");
2040        assert_eq!(resp.notes, "All checks pass");
2041        assert_eq!(resp.test_results.as_deref(), Some("pass"));
2042        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2043    }
2044
2045    #[serial(cooldown_state)]
2046    #[test]
2047    fn fixture_claude_review_request_changes() {
2048        let raw = include_str!("../../../tests/fixtures/review_claude_request_changes.jsonl");
2049        let resp = parse_review_via_agent("claude", raw).unwrap();
2050        assert_eq!(resp.decision, "request_changes");
2051        assert_eq!(resp.notes, "Fix the null deref on line 42");
2052        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2053        assert_eq!(resp.issues.len(), 1);
2054        assert_eq!(resp.issues[0].file, "src/main.rs");
2055        assert_eq!(resp.issues[0].line, Some(42));
2056        assert_eq!(resp.issues[0].severity, "error");
2057    }
2058
2059    #[serial(cooldown_state)]
2060    #[test]
2061    fn fixture_claude_review_rate_limit_returns_error() {
2062        // find_claude_result on an is_error=true result returns Some with is_error=true.
2063        // The integration test verifies this is captured, not that it's a ReviewResponse.
2064        let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2065        let result = find_agent_result("claude", raw);
2066        assert!(
2067            result.is_some(),
2068            "should find result even for error envelope"
2069        );
2070        let result = result.unwrap();
2071        assert!(result.is_error, "rate limit should be flagged as error");
2072        assert!(result.result_text.contains("rate limit"));
2073    }
2074
2075    // ── OpenCode review fixtures ────────────────────────────────────────────────
2076
2077    #[serial(cooldown_state)]
2078    #[test]
2079    fn fixture_opencode_review_approve() {
2080        let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2081        let resp = parse_review_via_agent("opencode", raw).unwrap();
2082        assert_eq!(resp.decision, "approve");
2083        assert_eq!(resp.notes, "All checks pass");
2084        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2085    }
2086
2087    #[serial(cooldown_state)]
2088    #[test]
2089    fn fixture_opencode_review_request_changes() {
2090        let raw = include_str!("../../../tests/fixtures/review_opencode_request_changes.jsonl");
2091        let resp = parse_review_via_agent("opencode", raw).unwrap();
2092        assert_eq!(resp.decision, "request_changes");
2093        assert_eq!(resp.notes, "Fix the memory leak");
2094        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2095        assert_eq!(resp.issues.len(), 1);
2096        assert_eq!(resp.issues[0].file, "src/lib.rs");
2097        assert_eq!(resp.issues[0].line, Some(100));
2098    }
2099
2100    #[serial(cooldown_state)]
2101    #[test]
2102    fn fixture_opencode_review_plain_text_approve() {
2103        // OpenCode with plain-text review (no JSON) — should infer approval via keywords.
2104        let raw = include_str!("../../../tests/fixtures/review_opencode_plain_text.jsonl");
2105        let resp = parse_review_via_agent("opencode", raw).unwrap();
2106        assert_eq!(resp.decision, "approve");
2107    }
2108
2109    // ── Codex review fixtures ────────────────────────────────────────────────────
2110
2111    #[serial(cooldown_state)]
2112    #[test]
2113    fn fixture_codex_review_approve() {
2114        let raw = include_str!("../../../tests/fixtures/review_codex_approve.jsonl");
2115        let resp = parse_review_via_agent("codex", raw).unwrap();
2116        assert_eq!(resp.decision, "approve");
2117        assert_eq!(resp.notes, "Clean implementation");
2118        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2119    }
2120
2121    #[serial(cooldown_state)]
2122    #[test]
2123    fn fixture_codex_review_request_changes() {
2124        let raw = include_str!("../../../tests/fixtures/review_codex_request_changes.jsonl");
2125        let resp = parse_review_via_agent("codex", raw).unwrap();
2126        assert_eq!(resp.decision, "request_changes");
2127        assert_eq!(resp.notes, "Race condition on line 15");
2128        assert_eq!(resp.issues.len(), 1);
2129        assert_eq!(resp.issues[0].file, "src/server.rs");
2130        assert_eq!(resp.issues[0].line, Some(15));
2131    }
2132
2133    #[serial(cooldown_state)]
2134    #[test]
2135    fn fixture_codex_review_plain_text_approve() {
2136        // Codex with plain-text review — should infer approval via "LGTM" keyword.
2137        let raw = include_str!("../../../tests/fixtures/review_codex_plain_text.jsonl");
2138        let resp = parse_review_via_agent("codex", raw).unwrap();
2139        assert_eq!(resp.decision, "approve");
2140    }
2141
2142    // ── Kimi review fixtures ────────────────────────────────────────────────────
2143
2144    #[serial(cooldown_state)]
2145    #[test]
2146    fn fixture_kimi_review_approve() {
2147        let raw = include_str!("../../../tests/fixtures/review_kimi_approve.jsonl");
2148        let resp = parse_review_via_agent("kimi", raw).unwrap();
2149        assert_eq!(resp.decision, "approve");
2150        assert_eq!(resp.notes, "LGTM");
2151        assert_eq!(resp.test_results.as_deref(), Some("pass"));
2152        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2153    }
2154
2155    // ── MiniMax review fixtures ────────────────────────────────────────────────
2156
2157    #[serial(cooldown_state)]
2158    #[test]
2159    fn fixture_minimax_review_request_changes() {
2160        let raw = include_str!("../../../tests/fixtures/review_minimax_request_changes.jsonl");
2161        let resp = parse_review_via_agent("minimax", raw).unwrap();
2162        assert_eq!(resp.decision, "request_changes");
2163        assert_eq!(resp.notes, "Fix the bug");
2164        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2165        assert_eq!(resp.issues.len(), 1);
2166        assert_eq!(resp.issues[0].file, "src/foo.rs");
2167        assert_eq!(resp.issues[0].line, Some(42));
2168        assert_eq!(resp.issues[0].severity, "error");
2169    }
2170
2171    // ── Cross-agent token/metadata extraction ───────────────────────────────────
2172
2173    /// Verify that find_agent_result extracts token counts from each agent's format.
2174    #[serial(cooldown_state)]
2175    #[test]
2176    fn find_agent_result_extracts_tokens_claude() {
2177        let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2178        let result = find_agent_result("claude", raw).expect("should find result");
2179        assert_eq!(result.input_tokens, Some(200));
2180        assert_eq!(result.output_tokens, Some(30));
2181        assert!(result.cost_usd.is_none()); // review fixtures don't include cost
2182    }
2183
2184    #[serial(cooldown_state)]
2185    #[test]
2186    fn find_agent_result_extracts_tokens_opencode() {
2187        let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2188        let result = find_agent_result("opencode", raw).expect("should find result");
2189        assert_eq!(result.input_tokens, Some(2900));
2190        assert_eq!(result.output_tokens, Some(100));
2191        assert_eq!(result.cost_usd, Some(0.002));
2192    }
2193
2194    /// Verify that find_agent_result returns None for empty/unparseable input.
2195    #[serial(cooldown_state)]
2196    #[test]
2197    fn find_agent_result_returns_none_for_empty() {
2198        assert!(find_agent_result("claude", "").is_none());
2199        assert!(find_agent_result("opencode", "").is_none());
2200        assert!(find_agent_result("codex", "").is_none());
2201        assert!(find_agent_result("kimi", "").is_none());
2202        assert!(find_agent_result("minimax", "").is_none());
2203    }
2204
2205    #[serial(cooldown_state)]
2206    #[test]
2207    fn find_agent_result_returns_none_for_plain_text() {
2208        assert!(find_agent_result("claude", "just some plain text").is_none());
2209        assert!(find_agent_result("codex", "just some plain text").is_none());
2210        // OpenCode is a special case — it can sometimes return a result for plain text.
2211        // This is acceptable because opencode's extract_ndjson_text falls back gracefully.
2212    }
2213
2214    // ── Error propagation via find_agent_result ─────────────────────────────────
2215
2216    /// Verify that find_agent_result surfaces rate-limit errors from Claude.
2217    #[serial(cooldown_state)]
2218    #[test]
2219    fn find_agent_result_propagates_claude_rate_limit() {
2220        let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2221        let result = find_agent_result("claude", raw).expect("should find result");
2222        assert!(result.is_error, "should be flagged as error");
2223        assert!(result.result_text.contains("rate limit"));
2224    }
2225
2226    /// Two fake agents on PATH, a config that names them and a 1 ms backoff. Each nextest test
2227    /// runs in its own process, so the environment is not restored
2228    #[cfg(unix)]
2229    fn failover_env() -> tempfile::TempDir {
2230        let dir = tempfile::tempdir().unwrap();
2231        let bin = dir.path().join("bin");
2232        fs::create_dir(&bin).unwrap();
2233        for agent in ["claude", "codex"] {
2234            let path = bin.join(agent);
2235            fs::write(&path, "#!/bin/sh\n").unwrap();
2236            fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap();
2237        }
2238        let home = dir.path().join("home");
2239        fs::create_dir(&home).unwrap();
2240        fs::write(
2241            home.join("config.yml"),
2242            "agents: [claude, codex]\ndispatch:\n  retry_base_delay_ms: 1\n  retry_max_delay_ms: 1\nworkflow:\n  max_timeout_retries: 3\n",
2243        )
2244        .unwrap();
2245        let mut path = bin.as_os_str().to_os_string();
2246        path.push(":");
2247        path.push(env::var_os("PATH").unwrap_or_default());
2248        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2249        unsafe { env::set_var("PATH", path) };
2250        // SAFETY: same as above
2251        unsafe { env::set_var("ORCH_HOME", &home) };
2252        cmd_cache::reset_for_test();
2253        dir
2254    }
2255
2256    async fn failover_task() -> (Arc<TaskStore>, String) {
2257        let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
2258        let id = store
2259            .create_internal("owner/repo", "t", "b", "test", "", None)
2260            .await
2261            .unwrap();
2262        (store, format!("internal:{id}"))
2263    }
2264
2265    /// Every other agent is in cooldown, so the timed-out task retries from `new` after the backoff
2266    #[cfg(unix)]
2267    #[serial(cooldown_state)]
2268    #[tokio::test]
2269    async fn timeout_with_every_other_agent_cooled_retries_as_new() {
2270        let _env = failover_env();
2271        cooldown::reset_global_state().await;
2272        set_agent_cooldown("codex", 600).await;
2273        let (store, task_id) = failover_task().await;
2274        let store = Some(store);
2275
2276        let outcome = handle_failover(
2277            &task_id,
2278            "claude",
2279            RetryableError::Timeout,
2280            "claude timed out after 1804s",
2281            false,
2282            &store,
2283            "owner/repo",
2284        )
2285        .await;
2286
2287        assert_eq!(outcome.status, "new");
2288        assert!(outcome.next_agent.is_none());
2289        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2290            .await
2291            .unwrap();
2292        assert!(task.limit_reroute_chain.is_empty(), "chain must be cleared");
2293        assert!(
2294            task.last_error.contains("will retry"),
2295            "{}",
2296            task.last_error
2297        );
2298        assert_eq!(task.timeout_retries, 1);
2299    }
2300
2301    /// Another agent is free, so the timeout still reroutes to it
2302    #[cfg(unix)]
2303    #[serial(cooldown_state)]
2304    #[tokio::test]
2305    async fn timeout_with_a_free_agent_reroutes_to_it() {
2306        let _env = failover_env();
2307        cooldown::reset_global_state().await;
2308        let (store, task_id) = failover_task().await;
2309        let store = Some(store);
2310
2311        let outcome = handle_failover(
2312            &task_id,
2313            "claude",
2314            RetryableError::Timeout,
2315            "claude timed out after 1804s",
2316            false,
2317            &store,
2318            "owner/repo",
2319        )
2320        .await;
2321
2322        assert_eq!(outcome.status, "new");
2323        assert_eq!(outcome.next_agent.as_deref(), Some("codex"));
2324        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2325            .await
2326            .unwrap();
2327        assert_eq!(task.limit_reroute_chain, "claude");
2328        assert_eq!(task.agent.as_deref(), Some("codex"));
2329        assert!(
2330            task.model.as_deref().unwrap_or("").is_empty(),
2331            "model must be cleared so routing picks one for the fallback, got {:?}",
2332            task.model
2333        );
2334        assert!(
2335            task.last_error.contains("switching to codex"),
2336            "{}",
2337            task.last_error
2338        );
2339    }
2340
2341    /// The run_with_context reroute clear must keep the fallback while status is still InProgress
2342    #[cfg(unix)]
2343    #[serial(cooldown_state)]
2344    #[tokio::test]
2345    async fn timeout_failover_keeps_fallback_after_reroute_clear() {
2346        let _env = failover_env();
2347        cooldown::reset_global_state().await;
2348        let (store_arc, task_id) = failover_task().await;
2349        let store_id = store_arc
2350            .resolve_task_id("owner/repo", &task_id)
2351            .await
2352            .unwrap()
2353            .unwrap();
2354        store_arc
2355            .update_status(store_id, TaskStatus::InProgress)
2356            .await
2357            .unwrap();
2358        store_arc
2359            .set_fields(store_id, &[("agent", serde_json::json!("claude"))])
2360            .await
2361            .unwrap();
2362
2363        let store = Some(Arc::clone(&store_arc));
2364        let outcome = handle_failover(
2365            &task_id,
2366            "claude",
2367            RetryableError::Timeout,
2368            "claude timed out after 1804s",
2369            false,
2370            &store,
2371            "owner/repo",
2372        )
2373        .await;
2374        assert_eq!(outcome.status, "new");
2375        assert_eq!(outcome.next_agent.as_deref(), Some("codex"));
2376
2377        let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store_arc));
2378        runner
2379            .clear_agent_on_reroute_unless_fallback(&task_id, "claude")
2380            .await;
2381
2382        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2383            .await
2384            .unwrap();
2385        assert_eq!(task.agent.as_deref(), Some("codex"));
2386        assert_eq!(task.status, TaskStatus::InProgress);
2387        assert!(
2388            task.model.as_deref().unwrap_or("").is_empty(),
2389            "model must stay empty so routing picks one for the fallback, got {:?}",
2390            task.model
2391        );
2392
2393        let mut config = RouterConfig::default();
2394        let medium = config.model_map.entry("medium".to_string()).or_default();
2395        medium.insert("claude".to_string(), vec!["haiku".to_string()]);
2396        medium.insert("codex".to_string(), vec!["gpt-test".to_string()]);
2397        let agents = vec!["claude".to_string(), "codex".to_string()];
2398        let mut router = Router::new_for_test(config, agents);
2399        let labeled = ExternalTask {
2400            id: ExternalId(task_id.clone()),
2401            title: "Test".to_string(),
2402            body: "Test body".to_string(),
2403            state: "open".to_string(),
2404            labels: vec!["agent:claude".to_string()],
2405            author: "test".to_string(),
2406            created_at: "2026-01-01T00:00:00Z".to_string(),
2407            updated_at: "2026-01-01T00:00:00Z".to_string(),
2408            url: format!("https://github.com/owner/repo/issues/{task_id}"),
2409        };
2410        let result = router
2411            .route(&labeled, &store_arc, "owner/repo")
2412            .await
2413            .unwrap();
2414        assert_eq!(result.agent, "codex");
2415    }
2416
2417    /// A task that times out on every run blocks after `max_timeout_retries` and keeps its issue open
2418    #[cfg(unix)]
2419    #[serial(cooldown_state)]
2420    #[tokio::test]
2421    async fn repeated_timeouts_block_the_task() {
2422        let _env = failover_env();
2423        cooldown::reset_global_state().await;
2424        set_agent_cooldown("codex", 600).await;
2425        let (store, task_id) = failover_task().await;
2426        let store = Some(store);
2427
2428        let mut statuses = Vec::new();
2429        for _ in 0..3 {
2430            statuses.push(
2431                handle_failover(
2432                    &task_id,
2433                    "claude",
2434                    RetryableError::Timeout,
2435                    "claude timed out after 1804s",
2436                    false,
2437                    &store,
2438                    "owner/repo",
2439                )
2440                .await
2441                .status,
2442            );
2443        }
2444
2445        assert_eq!(statuses, ["new", "new", "blocked"]);
2446        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2447            .await
2448            .unwrap();
2449        assert_eq!(task.block_reason.as_deref(), Some("timed out 3 times"));
2450        assert_eq!(task.timeout_retries, 3);
2451    }
2452}