Skip to main content

orch/engine/runner/
response.rs

1//! Response collection + error classification.
2//!
3//! After the agent finishes (tmux session ends), this module:
4//! 1. Reads the output file
5//! 2. Parses the response JSON
6//! 3. Classifies errors (timeout, usage limit, auth, tooling)
7//! 4. Determines next action (success, reroute, needs_review)
8
9use crate::store;
10use crate::store::TaskStore;
11use std::collections::HashSet;
12use std::collections::hash_map::DefaultHasher;
13use std::fs;
14use std::fs::File;
15use std::hash::Hash;
16use std::hash::Hasher;
17use std::io::Read;
18use std::io::Seek;
19use std::io::SeekFrom;
20use std::path::{Path, PathBuf};
21use std::result;
22use std::sync::Arc;
23use std::thread;
24use std::time::Duration;
25use std::time::SystemTime;
26
27use tokio::fs as tokio_fs;
28use tokio::task;
29use tokio::time;
30// Cooldown tracking is implemented in `crate::engine::cooldown` (shared with the router).
31use crate::cli::ndjson;
32use crate::cmd_cache;
33use crate::config as app_config;
34use crate::engine;
35pub use crate::engine::cooldown::{
36    clear_expired_cooldowns, is_agent_in_cooldown, is_model_in_cooldown,
37    record_agent_failure_with_message, record_model_failure, set_agent_cooldown,
38};
39use crate::engine::router::config;
40use crate::home;
41use crate::parser::AgentResponse;
42use crate::store::MemoryEntry;
43use crate::store::Task;
44
45const JITTER_FACTOR: f64 = 0.3;
46
47/// Maximum output file size to read (10 MB).
48/// Prevents runaway outputs from consuming excessive memory.
49const MAX_OUTPUT_SIZE_BYTES: u64 = 10 * 1024 * 1024;
50
51/// Read a file with a size limit, returning truncated content if the file is too large.
52///
53/// This prevents runaway agent outputs from causing memory issues. If the file
54/// exceeds the limit, reads from the end of the file to capture the most recent
55/// output (which is typically where the result or error message is).
56fn read_file_with_limit<P: AsRef<Path>>(path: P, max_bytes: u64) -> anyhow::Result<String> {
57    let path = path.as_ref();
58    let metadata = fs::metadata(path)?;
59
60    if !metadata.is_file() {
61        anyhow::bail!("not a file: {}", path.display());
62    }
63
64    let file_size = metadata.len();
65
66    // If file is within limits, read the whole thing
67    if file_size <= max_bytes {
68        return fs::read_to_string(path).map_err(Into::into);
69    }
70
71    // File is too large - read from the end to get the most recent output
72    let mut file = File::open(path)?;
73    let start_pos = file_size.saturating_sub(max_bytes);
74    file.seek(SeekFrom::Start(start_pos))?;
75
76    let mut buffer = Vec::with_capacity(usize::try_from(max_bytes).unwrap_or(0));
77    file.read_to_end(&mut buffer)?;
78
79    // Try to convert to UTF-8, replacing invalid sequences
80    let content = String::from_utf8_lossy(&buffer);
81
82    // Log a warning about truncation
83    tracing::warn!(
84        path = %path.display(),
85        file_size = file_size,
86        max_bytes = max_bytes,
87        "output file truncated: exceeded size limit"
88    );
89
90    Ok(content.into_owned())
91}
92
93/// Calculate exponential backoff with jitter for fallback retries.
94/// Uses retry count from the reroute chain to determine the delay.
95///
96/// Delay = min(base * 2^retry_count, max) * (1 + jitter)
97pub(crate) fn calculate_backoff_delay(retry_count: usize) -> u64 {
98    let base_delay = config::retry_base_delay_ms();
99    let max_delay = config::retry_max_delay_ms();
100
101    // Exponential: base * 2^retry_count
102    let exponential =
103        base_delay * (2_u64.saturating_pow(u32::try_from(retry_count).unwrap_or(u32::MAX)));
104    let capped = exponential.min(max_delay);
105
106    // Add jitter: ±30% using a hash-based pseudo-random value.
107    // Mix system time (nanoseconds) with thread ID so concurrent retries on
108    // different threads get different delays even when called in the same
109    // microsecond, avoiding the thundering-herd problem.
110    #[expect(
111        clippy::cast_precision_loss,
112        clippy::cast_possible_truncation,
113        clippy::cast_sign_loss,
114        reason = "a jitter range in milliseconds, far below 2^52 and never negative"
115    )]
116    let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
117    let jitter = {
118        let mut h = DefaultHasher::new();
119        SystemTime::now().hash(&mut h);
120        thread::current().id().hash(&mut h);
121        h.finish() % (jitter_range * 2 + 1)
122    };
123
124    capped.saturating_sub(jitter_range) + jitter
125}
126
127fn fallback_retry_count(task: Option<&Task>) -> usize {
128    task.map_or(0, |t| {
129        let chain_retry_count = t
130            .limit_reroute_chain
131            .split(',')
132            .filter(|entry| !entry.is_empty())
133            .count();
134        let network_retry_count = usize::try_from(t.network_retries).unwrap_or(0);
135        chain_retry_count.max(network_retry_count)
136    })
137}
138
139/// Wait with exponential backoff before retrying with a fallback agent.
140/// Returns the delay applied in milliseconds.
141pub async fn wait_for_fallback_backoff(
142    task_id: &str,
143    store: &Option<Arc<TaskStore>>,
144    repo: &str,
145) -> u64 {
146    let task = store::opt_store_get_task(store, repo, task_id).await;
147
148    // Normal failover uses reroute chain length. NetworkError retries bypass
149    // failover, so track their own streak and use whichever retry source is larger.
150    let retry_count = fallback_retry_count(task.as_ref());
151
152    let delay = calculate_backoff_delay(retry_count);
153
154    if delay > 0 {
155        tracing::debug!(
156            task_id,
157            retry_count = retry_count,
158            delay_ms = delay,
159            "fallback backoff: waiting before retry"
160        );
161        time::sleep(Duration::from_millis(delay)).await;
162    }
163
164    delay
165}
166
167/// Outcome signal for the engine to update router weights.
168///
169/// Returned by the task runner so the engine can feed rate limit and
170/// success signals back to the router's weighted round-robin implementation.
171#[derive(Debug, Clone)]
172pub enum WeightSignal {
173    /// Agent completed a task successfully.
174    Success { agent: String },
175    /// Agent hit a rate limit / usage limit.
176    RateLimited { agent: String },
177    /// The run failed (timeout, no fallback, exhausted agents) and ended in `needs_review`.
178    /// Never read as a completed run, so the no-PR shortcut cannot mark the task done.
179    Failed { agent: String },
180    /// Task is blocked on child tasks (delegations). Do not trigger review agent.
181    Blocked,
182    /// Task was rerouted (silence detection, timeout, parse error) — reset to new for re-routing.
183    /// Distinct from None so the engine never converts a silent-reset task into needs_review.
184    Rerouted,
185    /// No weight-relevant signal (guard skipped task, unrecognised status, etc.)
186    None,
187}
188
189/// Read the agent's output file, trying multiple locations.
190///
191/// Checks per-task attempt directory first, then legacy flat paths.
192/// Output is truncated to `MAX_OUTPUT_SIZE_BYTES` to prevent runaway memory usage.
193pub async fn read_output_file(task_id: &str, primary_path: &Path, repo: &str) -> String {
194    // Primary: explicit output file (already points to attempt dir)
195    if let Some(content) = task::spawn_blocking({
196        let p = primary_path.to_path_buf();
197        move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
198    })
199    .await
200    .ok()
201    .and_then(result::Result::ok)
202        && !content.is_empty()
203    {
204        return content;
205    }
206
207    // Fallback: check all attempt dirs for this task (newest first)
208    if let Ok(task_dir) = home::task_dir_async(repo, task_id).await {
209        let attempts_dir = task_dir.join("attempts");
210        // Use async metadata for is_dir check
211        if tokio_fs::metadata(&attempts_dir)
212            .await
213            .is_ok_and(|m| m.is_dir())
214        {
215            let mut attempt_nums: Vec<u32> = task::spawn_blocking({
216                let a = attempts_dir.clone();
217                move || {
218                    match fs::read_dir(&a) {
219                        Ok(entries) => {
220                            let mut nums = Vec::new();
221                            for entry in entries {
222                                match entry {
223                                    Ok(e) => {
224                                        if let Some(n) =
225                                            e.file_name().to_str().and_then(|n| n.parse().ok())
226                                        {
227                                            nums.push(n);
228                                        }
229                                    }
230                                    Err(e) => tracing::warn!(
231                                        path = %a.display(),
232                                        err = %e,
233                                        "error reading attempt dir entry in output fallback"
234                                    ),
235                                }
236                            }
237                            nums
238                        }
239                        Err(e) => {
240                            tracing::warn!(path = %a.display(), err = %e, "failed to read attempts dir in output fallback");
241                            Vec::new()
242                        }
243                    }
244                }
245            })
246            .await
247            .unwrap_or_default();
248
249            attempt_nums.sort_unstable_by(|a, b| b.cmp(a)); // newest first
250            for n in attempt_nums {
251                let p = attempts_dir.join(n.to_string()).join("output.json");
252                if let Some(content) = task::spawn_blocking({
253                    let p = p.clone();
254                    move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
255                })
256                .await
257                .ok()
258                .and_then(result::Result::ok)
259                    && !content.is_empty()
260                {
261                    tracing::info!(task_id, path = %p.display(), "read output from attempt dir");
262                    return content;
263                }
264            }
265        }
266    }
267
268    // Legacy fallback locations
269    let state_dir = home::state_dir().unwrap_or_else(|_| PathBuf::from("/tmp"));
270
271    let mut fallbacks = vec![
272        PathBuf::from(format!("/tmp/output-{task_id}.json")),
273        state_dir.join(format!("output-{task_id}.json")),
274    ];
275
276    if let Ok(legacy_path) = home::state_file(&format!("output-{task_id}.json"))
277        && !fallbacks.contains(&legacy_path)
278    {
279        fallbacks.push(legacy_path);
280    }
281
282    for path in &fallbacks {
283        if let Some(content) = task::spawn_blocking({
284            let p = path.clone();
285            move || read_file_with_limit(p, MAX_OUTPUT_SIZE_BYTES)
286        })
287        .await
288        .ok()
289        .and_then(result::Result::ok)
290            && !content.is_empty()
291        {
292            tracing::info!(task_id, path = %path.display(), "read output from legacy fallback");
293            return content;
294        }
295    }
296
297    String::new()
298}
299
300/// Pick a fallback agent, avoiding agents already in the reroute chain and agents in cooldown.
301pub fn pick_fallback_agent(
302    current_agent: &str,
303    chain: &str,
304    available_agents: &[String],
305) -> Option<String> {
306    // Clear expired cooldowns first
307    clear_expired_cooldowns();
308
309    let chain_set: HashSet<&str> = if chain.is_empty() {
310        HashSet::new()
311    } else {
312        chain.split(',').collect()
313    };
314
315    // Prefer agents that are not in cooldown and are not heavily degraded
316    // from recent rate-limits. This prevents immediate failover to agents
317    // that have been heavily penalized and likely to fail as well.
318    for agent in available_agents {
319        if agent == current_agent || chain_set.contains(agent.as_str()) {
320            continue;
321        }
322        if is_agent_in_cooldown(agent) {
323            continue;
324        }
325        // Weight-based filtering was removed from this module because it used
326        // a default-constructed AgentWeights (dead code that always returned
327        // true). Keep failover behavior simple here: prefer the first agent
328        // that is not in cooldown and not in the reroute chain. If we later
329        // expose the router's AgentWeights snapshot to this module we can add
330        // weight-based avoidance back in a non-dead way.
331        return Some(agent.clone());
332    }
333
334    None
335}
336
337/// Retryable error types that should trigger agent failover.
338#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub enum RetryableError {
340    /// Agent timed out.
341    Timeout,
342    /// Run exited without a result envelope
343    Interrupted,
344    /// Usage/rate limit hit.
345    UsageLimit,
346    /// Auth/billing error.
347    AuthError,
348    /// General failure (non-retryable but we still try fallback agents).
349    Failed,
350    /// Missing tooling - fallback might help if another agent has the tool.
351    MissingTooling,
352    /// A specific model was unavailable (e.g. "Model not found"). The agent
353    /// itself is fine — only the model is in cooldown. Failover may still pick
354    /// a different agent, but we must not bump the agent-level failure counter.
355    ModelUnavailable,
356}
357
358impl RetryableError {
359    /// Return a short classified string for this error type.
360    /// Used in DB rate-limit metrics and test assertions.
361    pub fn type_str(self) -> &'static str {
362        match self {
363            RetryableError::Timeout => "timeout",
364            RetryableError::Interrupted => "interrupted",
365            RetryableError::UsageLimit => "rate_limit",
366            RetryableError::AuthError => "auth_error",
367            RetryableError::ModelUnavailable => "model_unavailable",
368            RetryableError::Failed | RetryableError::MissingTooling => "failed",
369        }
370    }
371}
372
373/// Next task status from [`handle_failover`], plus the fallback agent when one was stored
374#[derive(Debug, Clone, PartialEq, Eq)]
375pub struct FailoverOutcome {
376    /// Task status to apply: `new`, `blocked`, or `needs_review`
377    pub status: String,
378    /// Fallback agent written to the task when failover switched
379    pub next_agent: Option<String>,
380}
381
382impl FailoverOutcome {
383    fn with_status(status: impl Into<String>) -> Self {
384        Self {
385            status: status.into(),
386            next_agent: None,
387        }
388    }
389
390    fn switched(next_agent: String) -> Self {
391        Self {
392            status: "new".to_string(),
393            next_agent: Some(next_agent),
394        }
395    }
396}
397
398/**
399 * Handle failover for any retryable error type. Status is "new" on reroute (or
400 * retryable exhaustion), "blocked" on hard exhaustion, "needs_review" when no
401 * fallback agents are available. `next_agent` is set when a fallback was stored
402 * so the next route prefers it over an `agent:*` label. `credit_recorded` means
403 * the caller already ran `record_credit_exhaustion`; the generic `failure_count:`
404 * counter must not be bumped here or the next ordinary failure starts at the
405 * extended backoff tier (issue #3652). Rate-limit event DB recording is done by
406 * the caller (mod.rs)
407 **/
408pub async fn handle_failover(
409    task_id: &str,
410    agent_name: &str,
411    error_type: RetryableError,
412    error_message: &str,
413    credit_recorded: bool,
414    store: &Option<Arc<TaskStore>>,
415    repo: &str,
416) -> FailoverOutcome {
417    // Get the reroute chain
418    let chain = get_reroute_chain(task_id, store, repo).await;
419    let chain = update_reroute_chain(task_id, agent_name, &chain, store, repo).await;
420
421    if error_type == RetryableError::Timeout
422        && let Some(status) =
423            block_after_repeated_timeouts(task_id, error_message, store, repo).await
424    {
425        return FailoverOutcome::with_status(status);
426    }
427
428    // Get all available agents
429    let available: Vec<String> = engine::configured_agents()
430        .into_iter()
431        .filter(|a| cmd_cache::command_exists(a))
432        .collect();
433
434    // Check if we've exhausted all agents (build chain_set once)
435    let chain_set: HashSet<&str> = if chain.is_empty() {
436        HashSet::new()
437    } else {
438        chain.split(',').collect()
439    };
440    let is_exhausted = available.iter().all(|a| chain_set.contains(a.as_str()));
441
442    if is_exhausted {
443        // Retryable failures (rate limits, quota, model unavailable) will resolve once
444        // cooldowns expire — reset to "new" so the router picks it up again then.
445        // Hard failures (generic agent crash, missing tooling) won't improve on retry;
446        // block immediately with a clear reason rather than wasting 63 minutes in the
447        // review-refire loop before getting a misleading "review agent rebroadcast" message.
448        let is_retryable = matches!(
449            error_type,
450            RetryableError::UsageLimit
451                | RetryableError::AuthError
452                | RetryableError::ModelUnavailable
453                | RetryableError::Timeout
454                | RetryableError::Interrupted
455        );
456        let next_status = if is_retryable { "new" } else { "blocked" };
457
458        tracing::warn!(
459            task_id,
460            agent = agent_name,
461            error_type = ?error_type,
462            next_status,
463            "all agents exhausted"
464        );
465
466        let mut store_fields: Vec<(&str, serde_json::Value)> = Vec::new();
467
468        if is_retryable {
469            // Clear reroute chain so the next dispatch builds a fresh one rather than
470            // immediately seeing "all agents in chain" and exhausting again.
471            let msg = format!("{error_message} (all agents exhausted, will retry when available)");
472            store_fields.push(("last_error", serde_json::json!(msg)));
473            store_fields.push(("limit_reroute_chain", serde_json::json!("")));
474        } else {
475            let msg = format!("{error_message} (all agents exhausted)");
476            let block_reason = format!(
477                "all agents exhausted: {} — {}",
478                error_type.type_str(),
479                error_message
480            );
481            store_fields.push(("last_error", serde_json::json!(msg)));
482            store_fields.push(("block_reason", serde_json::json!(block_reason)));
483        }
484
485        if let Err(e) = store::store_set_result(store, repo, task_id, &store_fields).await {
486            tracing::warn!(task_id, err = %e, "failed to write agents_exhausted fields to store");
487        }
488
489        // Log reroute activity for exhausted agents
490        let details = serde_json::json!({
491            "failure_reason": error_type.type_str(),
492            "error_message": error_message,
493            "chain": chain,
494            "agents_exhausted": true,
495            "is_retryable": is_retryable,
496        });
497        store::store_log_activity(
498            store,
499            repo,
500            task_id,
501            "rerouted",
502            Some("in_progress"),
503            Some(next_status),
504            None::<&str>,
505            None::<&str>,
506            Some(&details),
507        )
508        .await;
509
510        return FailoverOutcome::with_status(next_status);
511    }
512
513    // Pick a fallback agent
514    if let Some(next) = pick_fallback_agent(agent_name, &chain, &available) {
515        tracing::info!(
516            task_id,
517            from = agent_name,
518            to = next,
519            error_type = ?error_type,
520            "failover: switching to fallback agent"
521        );
522
523        // Timeouts and interrupts are not agent faults, skip the agent cooldown
524        if !matches!(
525            error_type,
526            RetryableError::MissingTooling
527                | RetryableError::ModelUnavailable
528                | RetryableError::Timeout
529                | RetryableError::Interrupted
530        ) {
531            // Credit errors skip the generic counter (already counted via
532            // credit_failure_count); the 120s cooldown below still applies
533            if !credit_recorded {
534                record_agent_failure_with_message(agent_name, error_message).await;
535            }
536
537            // Apply brief 120s cooldown on the failed agent so the router skips it
538            // on the next routing attempt. This prevents the router from immediately
539            // re-selecting the same agent after clearing both agent and model (#1492).
540            // Also skipped for ModelUnavailable: the agent is healthy.
541            set_agent_cooldown(agent_name, 120).await;
542        }
543
544        // Store the fallback agent so the next route prefers it over an agent:* label
545        // Clear the model so routing picks one that the fallback agent can run
546        let msg = format!("{error_message}, switching to {next} for re-route");
547        if let Err(e) = store::store_set_result(
548            store,
549            repo,
550            task_id,
551            &[
552                ("agent", serde_json::json!(next)),
553                ("model", serde_json::json!("")),
554                ("last_error", serde_json::json!(msg)),
555            ],
556        )
557        .await
558        {
559            tracing::warn!(task_id, err = %e, "failed to write failover agent to store");
560        }
561
562        // Apply exponential backoff with jitter before retry.
563        // Return "new" (not "routed") so the task goes through Phase 3a routing
564        // again. This ensures the router actually re-selects a compatible agent+model
565        // via model_for_complexity() instead of bypassing routing and defaulting to
566        // claude in the dispatch loop (#1604).
567        wait_for_fallback_backoff(task_id, store, repo).await;
568
569        // Log reroute activity for successful failover
570        let details = serde_json::json!({
571            "failure_reason": error_type.type_str(),
572            "error_message": error_message,
573            "from_agent": agent_name,
574            "to_agent": next,
575            "chain": chain,
576        });
577        store::store_log_activity(
578            store,
579            repo,
580            task_id,
581            "rerouted",
582            Some("in_progress"),
583            Some("new"),
584            Some(&next),
585            None::<&str>,
586            Some(&details),
587        )
588        .await;
589
590        return FailoverOutcome::switched(next);
591    }
592
593    // No fallback available
594    tracing::warn!(task_id, agent = agent_name, "no fallback agents available");
595
596    let is_retryable = matches!(
597        error_type,
598        RetryableError::UsageLimit
599            | RetryableError::AuthError
600            | RetryableError::ModelUnavailable
601            | RetryableError::Timeout
602            | RetryableError::Interrupted
603    );
604
605    // Timeouts and interrupts are not agent faults, skip the agent cooldown
606    if !matches!(
607        error_type,
608        RetryableError::MissingTooling
609            | RetryableError::ModelUnavailable
610            | RetryableError::Timeout
611            | RetryableError::Interrupted
612    ) && !credit_recorded
613    {
614        record_agent_failure_with_message(agent_name, error_message).await;
615    }
616
617    if is_retryable {
618        // The other agents are in cooldown, so wait out the backoff and let the router
619        // pick the task up again. A fresh chain keeps the next run from seeing every agent as tried
620        wait_for_fallback_backoff(task_id, store, repo).await;
621
622        let msg = format!("{error_message} (no fallback agents, will retry)");
623        if let Err(e) = store::store_set_result(
624            store,
625            repo,
626            task_id,
627            &[
628                ("last_error", serde_json::json!(msg)),
629                ("limit_reroute_chain", serde_json::json!("")),
630            ],
631        )
632        .await
633        {
634            tracing::warn!(task_id, err = %e, "failed to write no_fallback retry fields to store");
635        }
636
637        let details = serde_json::json!({
638            "failure_reason": error_type.type_str(),
639            "error_message": error_message,
640            "chain": chain,
641            "no_fallback_available": true,
642            "is_retryable": true,
643        });
644        store::store_log_activity(
645            store,
646            repo,
647            task_id,
648            "rerouted",
649            Some("in_progress"),
650            Some("new"),
651            None::<&str>,
652            None::<&str>,
653            Some(&details),
654        )
655        .await;
656
657        return FailoverOutcome::with_status("new");
658    }
659
660    let msg = format!("{error_message}, no fallback agents");
661    if let Err(e) = store::store_set_result(
662        store,
663        repo,
664        task_id,
665        &[("last_error", serde_json::json!(msg))],
666    )
667    .await
668    {
669        tracing::warn!(task_id, err = %e, "failed to write no_fallback last_error to store");
670    }
671
672    // Log reroute activity for no fallback available
673    let details = serde_json::json!({
674        "failure_reason": error_type.type_str(),
675        "error_message": error_message,
676        "chain": chain,
677        "no_fallback_available": true,
678    });
679    store::store_log_activity(
680        store,
681        repo,
682        task_id,
683        "rerouted",
684        Some("in_progress"),
685        Some("needs_review"),
686        None::<&str>,
687        None::<&str>,
688        Some(&details),
689    )
690    .await;
691
692    FailoverOutcome::with_status("needs_review")
693}
694
695/// Count a timeout against the task. Returns "blocked" once the task has timed out
696/// `workflow.max_timeout_retries` times (default 3), so a task that always times out does not loop
697pub async fn block_after_repeated_timeouts(
698    task_id: &str,
699    error_message: &str,
700    store: &Option<Arc<TaskStore>>,
701    repo: &str,
702) -> Option<String> {
703    let max: i64 = app_config::get("workflow.max_timeout_retries")
704        .ok()
705        .and_then(|v| v.parse().ok())
706        .unwrap_or(3);
707    let count = match store::store_increment(store, repo, task_id, "timeout_retries").await {
708        Ok(count) => i64::try_from(count).unwrap_or(i64::MAX),
709        Err(e) => {
710            tracing::warn!(task_id, err = %e, "failed to increment timeout_retries");
711            return None;
712        }
713    };
714    if count < max {
715        return None;
716    }
717
718    let block_reason = format!("timed out {count} times");
719    tracing::warn!(
720        task_id,
721        count,
722        max,
723        "timeout retries exhausted, blocking task"
724    );
725    if let Err(e) = store::store_set_result(
726        store,
727        repo,
728        task_id,
729        &[
730            (
731                "last_error",
732                serde_json::json!(format!("{error_message} ({block_reason})")),
733            ),
734            ("block_reason", serde_json::json!(block_reason)),
735            ("limit_reroute_chain", serde_json::json!("")),
736        ],
737    )
738    .await
739    {
740        tracing::warn!(task_id, err = %e, "failed to write timeout block fields to store");
741    }
742    let details = serde_json::json!({
743        "failure_reason": "timeout",
744        "error_message": error_message,
745        "timeout_retries": count,
746    });
747    store::store_log_activity(
748        store,
749        repo,
750        task_id,
751        "rerouted",
752        Some("in_progress"),
753        Some("blocked"),
754        None::<&str>,
755        None::<&str>,
756        Some(&details),
757    )
758    .await;
759    Some("blocked".to_string())
760}
761
762/// Get the reroute chain from store.
763pub async fn get_reroute_chain(
764    task_id: &str,
765    store: &Option<Arc<TaskStore>>,
766    repo: &str,
767) -> String {
768    store::opt_store_get_task(store, repo, task_id)
769        .await
770        .map(|t| t.limit_reroute_chain)
771        .unwrap_or_default()
772        .trim()
773        .to_string()
774}
775
776/// Update the reroute chain in store.
777pub async fn update_reroute_chain(
778    task_id: &str,
779    current_agent: &str,
780    existing_chain: &str,
781    store: &Option<Arc<TaskStore>>,
782    repo: &str,
783) -> String {
784    let mut chain = existing_chain.to_string();
785    if chain.is_empty() {
786        chain = current_agent.to_string();
787    } else if !chain.split(',').any(|a| a == current_agent) {
788        chain = format!("{chain},{current_agent}");
789    }
790
791    if let Err(e) = store::store_set_result(
792        store,
793        repo,
794        task_id,
795        &[("limit_reroute_chain", serde_json::json!(chain))],
796    )
797    .await
798    {
799        tracing::warn!(task_id, err = %e, "failed to write limit_reroute_chain to store");
800    }
801    chain
802}
803
804/// Review response from the review agent.
805#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
806pub struct ReviewResponse {
807    /// Decision: "approve" | "request_changes"
808    pub decision: String,
809    /// Detailed review feedback.
810    pub notes: String,
811    /// Test results: "pass" | "fail" | "skipped"
812    pub test_results: Option<String>,
813    /// List of issues found.
814    pub issues: Vec<ReviewIssue>,
815}
816
817/// A single issue found during review.
818#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
819pub struct ReviewIssue {
820    /// File path.
821    pub file: String,
822    /// Line number (optional).
823    pub line: Option<u32>,
824    /// Severity: "error" | "warning"
825    pub severity: String,
826    /// Description of the issue.
827    pub description: String,
828}
829
830/// Parse a review response from already-unwrapped text.
831///
832/// Expects the raw result text (already extracted from the agent envelope
833/// by the agent-specific parser). Tries direct JSON parse, then markdown
834/// code block extraction.
835pub fn parse_review_response(text: &str) -> anyhow::Result<ReviewResponse> {
836    let resp = parse_review_json(text)?;
837    if matches!(resp.decision.as_str(), "approve" | "request_changes") {
838        return Ok(resp);
839    }
840
841    // Weak models echo the prompt template ("approve|request_changes"), strip it so it
842    // does not match the "request_changes" keyword
843    let stripped = text.replace("approve|request_changes", "");
844    infer_review_response_from_text(&stripped)
845        .ok_or_else(|| anyhow::anyhow!("invalid review decision: {}", resp.decision))
846}
847
848fn parse_review_json(text: &str) -> anyhow::Result<ReviewResponse> {
849    // Try direct JSON parse
850    if let Ok(resp) = serde_json::from_str::<ReviewResponse>(text) {
851        return Ok(resp);
852    }
853
854    // Try extracting JSON from markdown code blocks
855    if let Some(json_str) = extract_json_block(text)
856        && let Ok(resp) = serde_json::from_str::<ReviewResponse>(&json_str)
857    {
858        return Ok(resp);
859    }
860
861    if let Some(resp) = extract_review_response_object(text) {
862        return Ok(resp);
863    }
864
865    anyhow::bail!("failed to parse review response")
866}
867
868fn infer_review_response_from_text(text: &str) -> Option<ReviewResponse> {
869    let lower = text.to_ascii_lowercase();
870    let has_changes_requested =
871        lower.contains("changes requested") || lower.contains("request_changes");
872    if has_changes_requested {
873        return Some(ReviewResponse {
874            decision: "request_changes".to_string(),
875            notes: "Inferred decision from plain-text review output.".to_string(),
876            test_results: None,
877            issues: Vec::new(),
878        });
879    }
880
881    // Approval only from an explicit decision token. "approved" and CI-evidence
882    // phrases ("all checks passed") are not a decision, a negation list cannot cover them
883    let positive_approval = lower
884        .split(|c: char| !c.is_ascii_alphabetic())
885        .any(|word| word == "lgtm")
886        || lower.contains("decision is `approve`")
887        || lower.contains("decision: `approve`")
888        || lower.contains("decision is approve")
889        || lower.contains("decision: approve");
890    if positive_approval {
891        return Some(ReviewResponse {
892            decision: "approve".to_string(),
893            notes: "Inferred decision from plain-text review output.".to_string(),
894            test_results: None,
895            issues: Vec::new(),
896        });
897    }
898
899    None
900}
901
902/// Public entry point for keyword-based review response inference.
903///
904/// Used by `review.rs` when `parse_review_response` fails on the extracted text.
905pub fn infer_review_response(text: &str) -> Option<ReviewResponse> {
906    infer_review_response_from_text(text)
907}
908
909/// Extract the first valid JSON object from markdown code blocks.
910///
911/// Searches for ```json blocks and returns the first one whose content
912/// starts with `{`, skipping non-JSON code blocks.
913fn extract_json_block(text: &str) -> Option<String> {
914    let mut search_from = 0;
915    while let Some(start) = text[search_from..].find("```json") {
916        let abs_start = search_from + start;
917        let after_tag = abs_start + "```json".len();
918        let Some(rest) = text[after_tag..].strip_prefix('\n') else {
919            search_from = after_tag;
920            continue;
921        };
922        let content_start = text.len() - rest.len();
923        let Some(end_offset) = rest.find("```") else {
924            search_from = content_start;
925            continue;
926        };
927        let end = content_start + end_offset;
928        let content = text[content_start..end].trim();
929        if content.starts_with('{') {
930            return Some(content.to_string());
931        }
932        search_from = end + 3;
933    }
934    None
935}
936
937/// Extract a ReviewResponse JSON object from arbitrary text.
938///
939/// Scans for balanced JSON objects and tries to parse them as ReviewResponse.
940fn extract_review_response_object(text: &str) -> Option<ReviewResponse> {
941    let mut start: Option<usize> = None;
942    let mut depth: i32 = 0;
943    let mut in_string = false;
944    let mut escape = false;
945
946    for (idx, ch) in text.char_indices() {
947        if start.is_some() {
948            if in_string {
949                if escape {
950                    escape = false;
951                } else if ch == '\\' {
952                    escape = true;
953                } else if ch == '"' {
954                    in_string = false;
955                }
956                continue;
957            }
958
959            match ch {
960                '"' => in_string = true,
961                '{' => depth += 1,
962                '}' => {
963                    depth -= 1;
964                    if depth == 0 {
965                        debug_assert!(
966                            start.is_some(),
967                            "start must be Some when depth returns to 0"
968                        );
969                        let start_idx = start.unwrap_or(0);
970                        let candidate = &text[start_idx..=idx];
971                        if let Ok(resp) = serde_json::from_str::<ReviewResponse>(candidate) {
972                            return Some(resp);
973                        }
974                        start = None;
975                    }
976                }
977                _ => {}
978            }
979            continue;
980        }
981
982        if ch == '{' {
983            start = Some(idx);
984            depth = 1;
985            in_string = false;
986            escape = false;
987        }
988    }
989
990    None
991}
992
993/// Extract learnings and store as memory for future attempts.
994#[expect(
995    clippy::too_many_arguments,
996    reason = "a memory entry records the task, attempt, agent, model and outcome"
997)]
998pub async fn store_learnings_from_response(
999    task_id: &str,
1000    attempt: u32,
1001    agent: &str,
1002    model: Option<&str>,
1003    response: &AgentResponse,
1004    error: Option<&str>,
1005    store: &Option<Arc<TaskStore>>,
1006    repo: &str,
1007    raw_stdout: &str,
1008) {
1009    // Use agent self-reported files; fall back to NDJSON tool-call extraction when empty.
1010    let files_modified = if response.files.is_empty() {
1011        let extracted = ndjson::extract_files_changed(raw_stdout);
1012        if !extracted.is_empty() {
1013            tracing::debug!(
1014                task_id,
1015                count = extracted.len(),
1016                "files_modified: fell back to NDJSON extraction (agent reported none)"
1017            );
1018        }
1019        extracted
1020    } else {
1021        response.files.clone()
1022    };
1023
1024    // Build the memory entry
1025    let entry = MemoryEntry {
1026        attempt,
1027        agent: agent.to_string(),
1028        model: model.map(String::from),
1029        learnings: response.learnings.clone(),
1030        error: error.map(String::from),
1031        files_modified,
1032        approach: response.summary.clone(),
1033        timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1034    };
1035
1036    if let Some(st) = store
1037        && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1038    {
1039        match st.append_memory(store_id, &entry).await {
1040            Err(e) => {
1041                tracing::warn!(task_id, error = ?e, "failed to store memory");
1042            }
1043            _ => {
1044                tracing::debug!(task_id, attempt, "stored memory for attempt");
1045            }
1046        }
1047    }
1048}
1049
1050/// Store a memory entry for a failed attempt (without a full AgentResponse).
1051pub async fn store_failure_memory(
1052    task_id: &str,
1053    attempt: u32,
1054    agent: &str,
1055    model: Option<&str>,
1056    error: &str,
1057    store: &Option<Arc<TaskStore>>,
1058    repo: &str,
1059) {
1060    let entry = MemoryEntry {
1061        attempt,
1062        agent: agent.to_string(),
1063        model: model.map(String::from),
1064        learnings: vec![],
1065        error: Some(error.to_string()),
1066        files_modified: vec![],
1067        approach: String::new(),
1068        timestamp: chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
1069    };
1070
1071    if let Some(st) = store
1072        && let Ok(Some(store_id)) = st.resolve_task_id(repo, task_id).await
1073    {
1074        match st.append_memory(store_id, &entry).await {
1075            Err(e) => {
1076                tracing::warn!(task_id, error = ?e, "failed to store failure memory");
1077            }
1078            _ => {
1079                tracing::debug!(task_id, attempt, "stored failure memory");
1080            }
1081        }
1082    }
1083}
1084
1085#[cfg(test)]
1086mod tests {
1087    use std::env;
1088    #[cfg(unix)]
1089    use std::os::unix::fs::PermissionsExt;
1090    use std::process;
1091    use std::sync::Arc;
1092
1093    use super::super::TaskRunner;
1094    use super::*;
1095    use crate::backends::{ExternalId, ExternalTask};
1096    use crate::engine::cooldown;
1097    use crate::engine::router::{Router, RouterConfig};
1098    use crate::engine::runner::agents::{AgentError, find_agent_result, patterns};
1099    use crate::store::TaskStatus;
1100    use serial_test::serial;
1101
1102    #[serial(cooldown_state)]
1103    #[test]
1104    fn calculate_backoff_delay_jitter_is_centered() {
1105        // Run many samples and verify the range is [0.7*capped, 1.3*capped].
1106        // We can't control `now` directly, but calling the function many times
1107        // exercises different microsecond values and lets us check bounds.
1108        let base = config::retry_base_delay_ms();
1109        let max = config::retry_max_delay_ms();
1110
1111        // retry_count=0 → exponential = base, capped = base (assuming base < max)
1112        let capped = base.min(max);
1113        #[expect(
1114            clippy::cast_precision_loss,
1115            clippy::cast_possible_truncation,
1116            clippy::cast_sign_loss,
1117            reason = "mirrors the jitter range in calculate_backoff_delay"
1118        )]
1119        let jitter_range = (capped as f64 * JITTER_FACTOR) as u64;
1120        let lo = capped.saturating_sub(jitter_range);
1121        let hi = capped + jitter_range;
1122
1123        for _ in 0..200 {
1124            let delay = calculate_backoff_delay(0);
1125            assert!(
1126                delay >= lo && delay <= hi,
1127                "delay {delay} out of expected range [{lo}, {hi}]"
1128            );
1129        }
1130    }
1131
1132    #[serial(cooldown_state)]
1133    #[test]
1134    fn fallback_retry_count_uses_larger_network_retry_streak() {
1135        let task = Task {
1136            id: 1,
1137            external_id: None,
1138            repo: "owner/repo".to_string(),
1139            origin: "internal".to_string(),
1140            title: "test".to_string(),
1141            body: String::new(),
1142            status: TaskStatus::New,
1143            source: String::new(),
1144            source_id: String::new(),
1145            author: String::new(),
1146            url: String::new(),
1147            labels: vec![],
1148            agent: None,
1149            model: None,
1150            complexity: "simple".to_string(),
1151            estimate: 0,
1152            route_reason: String::new(),
1153            agent_profile: String::new(),
1154            selected_skills: String::new(),
1155            route_attempts: 0,
1156            attempts: 0,
1157            branch: String::new(),
1158            worktree: String::new(),
1159            worktree_cleaned: false,
1160            owner_node: None,
1161            summary: String::new(),
1162            last_error: String::new(),
1163            parent_id: None,
1164            block_reason: None,
1165            pr_number: None,
1166            pr_review_context: String::new(),
1167            last_review_ts: String::new(),
1168            review_ts_map: "{}".to_string(),
1169            last_comment_review_ts: String::new(),
1170            merge_conflict_retries: 0,
1171            ci_merge_failures: 0,
1172            pr_create_failures: 0,
1173            push_failures: 0,
1174            network_retries: 3,
1175            timeout_retries: 0,
1176            review_agent_failures: 0,
1177            review_resets: 0,
1178            review_cycles: 0,
1179            review_invocations: 0,
1180            needs_review_refires: 0,
1181            input_tokens: 0,
1182            output_tokens: 0,
1183            total_cost_usd: 0.0,
1184            model_reroute_chain: String::new(),
1185            limit_reroute_chain: "claude".to_string(),
1186            memory: vec![],
1187            delegations: vec![],
1188            auto_unblock_count: 0,
1189            auto_unblock_last_at: String::new(),
1190            auto_unblock_last_reason: String::new(),
1191            ci_recovery_count: 0,
1192            no_code_reroutes: 0,
1193            no_code_last_agent: String::new(),
1194            created_at: String::new(),
1195            updated_at: String::new(),
1196        };
1197
1198        assert_eq!(fallback_retry_count(Some(&task)), 3);
1199    }
1200
1201    #[serial(cooldown_state)]
1202    #[test]
1203    fn retryable_type_str_returns_correct_labels() {
1204        assert_eq!(RetryableError::Timeout.type_str(), "timeout");
1205        assert_eq!(RetryableError::Interrupted.type_str(), "interrupted");
1206        assert_eq!(RetryableError::UsageLimit.type_str(), "rate_limit");
1207        assert_eq!(RetryableError::AuthError.type_str(), "auth_error");
1208        assert_eq!(RetryableError::Failed.type_str(), "failed");
1209        assert_eq!(RetryableError::MissingTooling.type_str(), "failed");
1210    }
1211
1212    #[serial(cooldown_state)]
1213    #[test]
1214    fn is_usage_limit_detects_rate_limit() {
1215        assert!(patterns::detect_rate_limit("Error: rate limit exceeded").is_some());
1216        assert!(patterns::detect_rate_limit("HTTP 429 Too Many Requests").is_some());
1217        assert!(patterns::detect_rate_limit("quota exceeded for model").is_some());
1218        // context_length_exceeded is handled by detect_context_overflow, not detect_rate_limit
1219        assert!(patterns::detect_context_overflow("context_length_exceeded").is_some());
1220    }
1221
1222    #[serial(cooldown_state)]
1223    #[test]
1224    fn is_usage_limit_rejects_normal_text() {
1225        assert!(patterns::detect_rate_limit("task completed successfully").is_none());
1226        assert!(patterns::detect_rate_limit("").is_none());
1227    }
1228
1229    #[serial(cooldown_state)]
1230    #[test]
1231    fn is_auth_error_detects_common_patterns() {
1232        assert!(patterns::detect_auth_error("401 Unauthorized").is_some());
1233        assert!(patterns::detect_auth_error("invalid api key provided").is_some());
1234        assert!(patterns::detect_auth_error("Your billing plan has expired").is_some());
1235        assert!(patterns::detect_auth_error("Error: 403 Forbidden").is_some());
1236    }
1237
1238    #[serial(cooldown_state)]
1239    #[test]
1240    fn is_auth_error_rejects_normal_text() {
1241        assert!(patterns::detect_auth_error("task completed successfully").is_none());
1242        assert!(patterns::detect_auth_error("").is_none());
1243    }
1244
1245    #[serial(cooldown_state)]
1246    #[test]
1247    fn detect_missing_tooling_finds_known_tools() {
1248        let result = patterns::detect_missing_tool("bun: command not found");
1249        assert!(result.is_some());
1250        if let AgentError::MissingTool { tool } = result.unwrap() {
1251            assert_eq!(tool, "bun");
1252        } else {
1253            panic!("expected MissingTool error");
1254        }
1255
1256        let result = patterns::detect_missing_tool("env: anchor: no such file");
1257        assert!(result.is_some());
1258        if let AgentError::MissingTool { tool } = result.unwrap() {
1259            assert_eq!(tool, "anchor");
1260        } else {
1261            panic!("expected MissingTool error");
1262        }
1263
1264        let result = patterns::detect_missing_tool("spawn docker enoent");
1265        assert!(result.is_some());
1266        if let AgentError::MissingTool { tool } = result.unwrap() {
1267            assert_eq!(tool, "docker");
1268        } else {
1269            panic!("expected MissingTool error");
1270        }
1271    }
1272
1273    #[serial(cooldown_state)]
1274    #[test]
1275    fn detect_missing_tooling_returns_none_for_normal() {
1276        assert!(patterns::detect_missing_tool("everything works fine").is_none());
1277        assert!(patterns::detect_missing_tool("").is_none());
1278    }
1279
1280    // ── Cooldown tests ────────────────────────────────────────────
1281
1282    #[serial(cooldown_state)]
1283    #[tokio::test]
1284    async fn record_and_check_agent_cooldown() {
1285        cooldown::reset_global_state().await;
1286        // Use unique names to avoid interference from other tests
1287        let agent = "test_cooldown_agent_1";
1288        assert!(!is_agent_in_cooldown(agent));
1289        record_agent_failure_with_message(agent, "").await;
1290        assert!(is_agent_in_cooldown(agent));
1291    }
1292
1293    #[serial(cooldown_state)]
1294    #[tokio::test]
1295    async fn record_and_check_model_cooldown() {
1296        cooldown::reset_global_state().await;
1297        let agent = "test_cooldown_agent_2";
1298        let model = "test_model_x";
1299        assert!(!is_model_in_cooldown(agent, model));
1300        record_model_failure(agent, model).await;
1301        assert!(is_model_in_cooldown(agent, model));
1302        // Different model should not be in cooldown
1303        assert!(!is_model_in_cooldown(agent, "other_model"));
1304    }
1305
1306    #[serial(cooldown_state)]
1307    #[tokio::test]
1308    async fn clear_expired_does_not_remove_fresh_entries() {
1309        cooldown::reset_global_state().await;
1310        let agent = "test_cooldown_agent_3";
1311        record_agent_failure_with_message(agent, "").await;
1312        clear_expired_cooldowns();
1313        // Should still be in cooldown (just recorded)
1314        assert!(is_agent_in_cooldown(agent));
1315    }
1316
1317    // Use fake agent names so other tests don't interfere.
1318    #[serial(cooldown_state)]
1319    #[test]
1320    fn pick_fallback_skips_current_agent() {
1321        let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1322        let result = pick_fallback_agent("test_agent_a", "", &available);
1323        assert_eq!(result, Some("test_agent_b".to_string()));
1324    }
1325
1326    #[serial(cooldown_state)]
1327    #[test]
1328    fn pick_fallback_skips_chain_agents() {
1329        let available = vec![
1330            "test_agent_a".to_string(),
1331            "test_agent_b".to_string(),
1332            "test_agent_c".to_string(),
1333        ];
1334        let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1335        assert_eq!(result, Some("test_agent_c".to_string()));
1336    }
1337
1338    #[serial(cooldown_state)]
1339    #[test]
1340    fn pick_fallback_returns_none_when_exhausted() {
1341        let available = vec!["test_agent_a".to_string(), "test_agent_b".to_string()];
1342        let result = pick_fallback_agent("test_agent_a", "test_agent_a,test_agent_b", &available);
1343        assert!(result.is_none());
1344    }
1345
1346    #[serial(cooldown_state)]
1347    #[test]
1348    fn weight_signal_variants() {
1349        let success = WeightSignal::Success {
1350            agent: "claude".to_string(),
1351        };
1352        let limited = WeightSignal::RateLimited {
1353            agent: "codex".to_string(),
1354        };
1355        let none = WeightSignal::None;
1356
1357        // Verify Debug trait
1358        assert!(format!("{success:?}").contains("claude"));
1359        assert!(format!("{limited:?}").contains("codex"));
1360        assert!(format!("{none:?}").contains("None"));
1361    }
1362
1363    #[serial(cooldown_state)]
1364    #[test]
1365    fn parse_review_response_direct_json() {
1366        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1367        let resp = parse_review_response(json).unwrap();
1368        assert_eq!(resp.decision, "approve");
1369        assert_eq!(resp.notes, "LGTM");
1370        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1371    }
1372
1373    #[serial(cooldown_state)]
1374    #[test]
1375    fn parse_review_response_from_markdown() {
1376        let md = r#"Here is my review:
1377
1378```json
1379{"decision":"request_changes","notes":"Fix the bug","issues":[{"file":"src/main.rs","line":10,"severity":"error","description":"null deref"}]}
1380```
1381
1382That's all."#;
1383        let resp = parse_review_response(md).unwrap();
1384        assert_eq!(resp.decision, "request_changes");
1385        assert_eq!(resp.issues.len(), 1);
1386        assert_eq!(resp.issues[0].file, "src/main.rs");
1387    }
1388
1389    #[serial(cooldown_state)]
1390    #[test]
1391    fn parse_review_response_embedded_json_object() {
1392        let text = "Review complete.\n\n{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}\nThanks!";
1393        let resp = parse_review_response(text).unwrap();
1394        assert_eq!(resp.decision, "approve");
1395        assert_eq!(resp.notes, "LGTM");
1396    }
1397
1398    #[serial(cooldown_state)]
1399    #[test]
1400    fn parse_review_response_template_echo_infers_from_notes() {
1401        let json = r#"{"decision":"approve|request_changes","notes":"LGTM","issues":[]}"#;
1402        assert_eq!(parse_review_response(json).unwrap().decision, "approve");
1403
1404        let no_signal = r#"{"decision":"approve|request_changes","notes":"hmm","issues":[]}"#;
1405        assert!(parse_review_response(no_signal).is_err());
1406    }
1407
1408    #[serial(cooldown_state)]
1409    #[test]
1410    fn parse_review_response_invalid() {
1411        let result = parse_review_response("not json at all");
1412        assert!(result.is_err());
1413    }
1414
1415    #[serial(cooldown_state)]
1416    #[test]
1417    fn extract_json_block_basic() {
1418        let md = "text\n```json\n{\"key\": \"value\"}\n```\nmore";
1419        let result = extract_json_block(md);
1420        assert!(result.is_some());
1421        assert!(result.unwrap().contains("\"key\""));
1422    }
1423
1424    #[serial(cooldown_state)]
1425    #[test]
1426    fn extract_json_block_skips_non_json_blocks() {
1427        let md = "```json\nnot-json-array\n```\n\n```json\n{\"real\": true}\n```";
1428        let result = extract_json_block(md);
1429        assert!(result.is_some());
1430        assert!(result.unwrap().contains("\"real\""));
1431    }
1432
1433    #[serial(cooldown_state)]
1434    #[test]
1435    fn extract_json_block_none_when_missing() {
1436        assert!(extract_json_block("no code blocks here").is_none());
1437    }
1438
1439    #[serial(cooldown_state)]
1440    #[test]
1441    fn extract_json_block_skips_malformed_intermediate_blocks() {
1442        let md = "prefix ```json{\"broken\": true}\n\n```json\n{\"real\": true}\n```";
1443        let result = extract_json_block(md);
1444        assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1445    }
1446
1447    #[serial(cooldown_state)]
1448    #[test]
1449    fn extract_json_block_skips_multiple_malformed_intermediate_blocks() {
1450        let md = "prefix ```json{\"broken\": true}\nmid ```json[1, 2, 3]\n\n```json\n{\"real\": true}\n```";
1451        let result = extract_json_block(md);
1452        assert_eq!(result.as_deref(), Some("{\"real\": true}"));
1453    }
1454
1455    // ── parse_review_response + infer_review_response (production path) ─
1456
1457    /// Helper that mirrors the production plain-text path in review.rs:
1458    /// parse_review_response → infer_review_response.
1459    fn parse_review_plain(text: &str) -> anyhow::Result<ReviewResponse> {
1460        parse_review_response(text).or_else(|_| {
1461            infer_review_response(text)
1462                .ok_or_else(|| anyhow::anyhow!("failed to parse review response"))
1463        })
1464    }
1465
1466    #[serial(cooldown_state)]
1467    #[test]
1468    fn parse_review_from_output_direct_json() {
1469        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1470        let resp = parse_review_response(json).unwrap();
1471        assert_eq!(resp.decision, "approve");
1472    }
1473
1474    #[serial(cooldown_state)]
1475    #[test]
1476    fn parse_review_from_output_markdown() {
1477        let md = "Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}\n```\n";
1478        let resp = parse_review_response(md).unwrap();
1479        assert_eq!(resp.decision, "request_changes");
1480    }
1481
1482    #[serial(cooldown_state)]
1483    #[test]
1484    fn parse_review_from_output_plain_text_fallback() {
1485        let text = "The review is done. LGTM.";
1486        let resp = parse_review_plain(text).unwrap();
1487        assert_eq!(resp.decision, "approve");
1488        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
1489    }
1490
1491    /// opencode NDJSON stream — text events use the `part.text` format.
1492    #[serial(cooldown_state)]
1493    #[test]
1494    fn parse_review_from_output_ndjson_part_format() {
1495        let ndjson = concat!(
1496            r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1497            "\n",
1498            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Reviewing the PR...\n\n```json\n{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1499            "\n",
1500            r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop"}}"#,
1501        );
1502        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1503        assert_eq!(resp.decision, "approve");
1504        assert_eq!(resp.notes, "All checks pass");
1505    }
1506
1507    /// opencode NDJSON stream — text events use the newer `text` directly-in-event format.
1508    #[serial(cooldown_state)]
1509    #[test]
1510    fn parse_review_from_output_ndjson_direct_text_format() {
1511        let ndjson = concat!(
1512            r#"{"type":"step_start","timestamp":1000,"sessionID":"ses_abc"}"#,
1513            "\n",
1514            r#"{"type":"text","timestamp":1001,"sessionID":"ses_abc","text":"Here is my review:\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the bug\",\"test_results\":\"fail\",\"issues\":[]}\n```\n"}"#,
1515            "\n",
1516            r#"{"type":"step_finish","timestamp":1002,"sessionID":"ses_abc"}"#,
1517        );
1518        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1519        assert_eq!(resp.decision, "request_changes");
1520        assert_eq!(resp.notes, "Fix the bug");
1521    }
1522
1523    /// opencode NDJSON with ReviewResponse JSON directly in a text event (no markdown wrapper).
1524    #[serial(cooldown_state)]
1525    #[test]
1526    fn parse_review_from_output_ndjson_direct_json_in_text_event() {
1527        let ndjson = concat!(
1528            r#"{"type":"step_start","timestamp":1000}"#,
1529            "\n",
1530            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1531            "\n",
1532            r#"{"type":"step_finish","timestamp":1002}"#,
1533        );
1534        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1535        assert_eq!(resp.decision, "approve");
1536    }
1537
1538    #[serial(cooldown_state)]
1539    #[test]
1540    fn parse_review_from_output_ndjson_prefers_final_json_text_event() {
1541        let ndjson = concat!(
1542            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Working on it..."}}"#,
1543            "\n",
1544            r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1545        );
1546        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1547        assert_eq!(resp.decision, "approve");
1548        assert_eq!(resp.notes, "LGTM");
1549    }
1550
1551    /// Concatenated text events produce a valid ReviewResponse.
1552    #[serial(cooldown_state)]
1553    #[test]
1554    fn parse_review_from_output_ndjson_concatenated_text() {
1555        let ndjson = concat!(
1556            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"Here is my\n"}}"#,
1557            "\n",
1558            r#"{"type":"text","timestamp":1002,"part":{"type":"text","text":"review:\n\n```json\n{\"decision\":\"approve\",\"notes\":\"OK\",\"test_results\":\"pass\",\"issues\":[]}\n```\n"}}"#,
1559        );
1560        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1561        assert_eq!(resp.decision, "approve");
1562    }
1563
1564    /// codex NDJSON stream — agent_message item carries the ReviewResponse JSON.
1565    ///
1566    /// Real format from codex `exec --json` output (observed 2026-03-06):
1567    /// thread.started / turn.started / item.completed(reasoning) / item.completed(agent_message) / turn.completed
1568    #[serial(cooldown_state)]
1569    #[test]
1570    fn parse_review_from_output_codex_agent_message() {
1571        let ndjson = concat!(
1572            r#"{"type":"thread.started","thread_id":"t1"}"#,
1573            "\n",
1574            r#"{"type":"turn.started"}"#,
1575            "\n",
1576            r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Planning git commands..."}}"#,
1577            "\n",
1578            r#"{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"git log --oneline -5","output":"abc123 fix: something"}}"#,
1579            "\n",
1580            r#"{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All checks pass\",\"test_results\":\"pass\",\"issues\":[]}"}}"#,
1581            "\n",
1582            r#"{"type":"turn.completed"}"#,
1583        );
1584        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1585        assert_eq!(resp.decision, "approve");
1586        assert_eq!(resp.notes, "All checks pass");
1587    }
1588
1589    /// codex NDJSON with ReviewResponse in a markdown code block inside agent_message.
1590    #[serial(cooldown_state)]
1591    #[test]
1592    fn parse_review_from_output_codex_agent_message_markdown() {
1593        let ndjson = concat!(
1594            r#"{"type":"thread.started","thread_id":"t1"}"#,
1595            "\n",
1596            r#"{"type":"turn.started"}"#,
1597            "\n",
1598            r#"{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"Reviewing..."}}"#,
1599            "\n",
1600            r#"{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Review complete.\n\n```json\n{\"decision\":\"request_changes\",\"notes\":\"Fix the tests\",\"issues\":[]}\n```\n"}}"#,
1601            "\n",
1602            r#"{"type":"turn.completed"}"#,
1603        );
1604        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1605        assert_eq!(resp.decision, "request_changes");
1606        assert_eq!(resp.notes, "Fix the tests");
1607    }
1608
1609    /// codex reasoning items must NOT be extracted as review text — only agent_message is used.
1610    #[serial(cooldown_state)]
1611    #[test]
1612    fn codex_skips_reasoning_items() {
1613        let ndjson = concat!(
1614            r#"{"type":"item.completed","item":{"type":"reasoning","text":"internal thoughts..."}}"#,
1615            "\n",
1616            r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}}"#,
1617        );
1618        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1619        assert_eq!(resp.decision, "approve");
1620        assert_eq!(resp.notes, "LGTM");
1621    }
1622
1623    /// Negation patterns must NOT trigger a false approval.
1624    #[serial(cooldown_state)]
1625    #[test]
1626    fn infer_review_response_negation_not_approved() {
1627        assert!(
1628            parse_review_plain("The changes were not approved.").is_err(),
1629            "\"not approved\" should not infer approval"
1630        );
1631    }
1632
1633    #[serial(cooldown_state)]
1634    #[test]
1635    fn infer_review_response_negation_unapproved() {
1636        assert!(
1637            parse_review_plain("Unapproved changes found in the diff.").is_err(),
1638            "\"unapproved\" should not infer approval"
1639        );
1640    }
1641
1642    #[serial(cooldown_state)]
1643    #[test]
1644    fn infer_review_response_negation_not_approving() {
1645        assert!(
1646            parse_review_plain("I am not approving this PR.").is_err(),
1647            "\"not approving\" should not infer approval"
1648        );
1649    }
1650
1651    #[serial(cooldown_state)]
1652    #[test]
1653    fn infer_review_response_negation_cannot_be_approved() {
1654        assert!(
1655            parse_review_plain("This PR cannot be approved until the race in tick.rs is fixed.")
1656                .is_err(),
1657            "\"cannot be approved\" should not infer approval"
1658        );
1659    }
1660
1661    #[serial(cooldown_state)]
1662    #[test]
1663    fn infer_review_response_negation_should_not_be_approved() {
1664        assert!(
1665            parse_review_plain("The diff breaks the build and should not be approved.").is_err(),
1666            "\"should not be approved\" should not infer approval"
1667        );
1668    }
1669
1670    #[serial(cooldown_state)]
1671    #[test]
1672    fn infer_review_response_negation_not_yet_approved() {
1673        assert!(
1674            parse_review_plain("This change is not yet approved.").is_err(),
1675            "\"not yet approved\" should not infer approval"
1676        );
1677    }
1678
1679    #[serial(cooldown_state)]
1680    #[test]
1681    fn infer_review_response_negation_wont_approve() {
1682        assert!(
1683            parse_review_plain("I won't approve this until tests cover the retry path.").is_err(),
1684            "\"won't approve\" should not infer approval"
1685        );
1686    }
1687
1688    #[serial(cooldown_state)]
1689    #[test]
1690    fn infer_review_response_bare_approved() {
1691        assert!(
1692            parse_review_plain("The PR was approved.").is_err(),
1693            "bare \"approved\" should not infer approval"
1694        );
1695    }
1696
1697    /// Positive plain-text approval signals must still work.
1698    #[serial(cooldown_state)]
1699    #[test]
1700    fn infer_review_response_lgtm() {
1701        let resp = parse_review_plain("LGTM, everything looks fine.").unwrap();
1702        assert_eq!(resp.decision, "approve");
1703    }
1704
1705    /// Evidence phrases are not a decision token
1706    #[serial(cooldown_state)]
1707    #[test]
1708    fn infer_review_response_looks_good() {
1709        assert!(
1710            parse_review_plain("Looks good to me!").is_err(),
1711            "\"looks good\" should not infer approval"
1712        );
1713    }
1714
1715    #[serial(cooldown_state)]
1716    #[test]
1717    fn infer_review_response_all_checks_passed() {
1718        assert!(
1719            parse_review_plain(
1720                "The background task completed but I already got the CI results directly. All checks passed — no action needed.",
1721            )
1722            .is_err(),
1723            "\"all checks passed\" should not infer approval"
1724        );
1725    }
1726
1727    #[serial(cooldown_state)]
1728    #[test]
1729    fn infer_review_response_checks_passed() {
1730        assert!(
1731            parse_review_plain("All checks passed (fmt ✓, clippy ✓, tests ✓).").is_err(),
1732            "\"all checks passed\" should not infer approval"
1733        );
1734    }
1735
1736    #[serial(cooldown_state)]
1737    #[test]
1738    fn infer_review_response_all_tests_passed() {
1739        assert!(
1740            parse_review_plain("All tests passed and the implementation looks correct.").is_err(),
1741            "\"all tests passed\" should not infer approval"
1742        );
1743    }
1744
1745    #[serial(cooldown_state)]
1746    #[test]
1747    fn infer_review_response_no_issues_found() {
1748        assert!(
1749            parse_review_plain("Review complete. No issues found in the diff.").is_err(),
1750            "\"no issues found\" should not infer approval"
1751        );
1752    }
1753
1754    /// Partial "tests passed" mid-sentence must NOT infer approval without "all" prefix.
1755    #[serial(cooldown_state)]
1756    #[test]
1757    fn infer_review_response_partial_tests_passed_no_approve() {
1758        let text = "The existing tests passed before this change but the new payment logic has a null-pointer risk on line 42.";
1759        assert!(parse_review_plain(text).is_err());
1760    }
1761
1762    /// Partial "checks passed" mid-sentence must NOT infer approval without "all" prefix.
1763    #[serial(cooldown_state)]
1764    #[test]
1765    fn infer_review_response_partial_checks_passed_no_approve() {
1766        let text = "CI checks passed for the base branch but this PR adds a broken path.";
1767        assert!(parse_review_plain(text).is_err());
1768    }
1769
1770    /// LLMs sometimes echo the JSON decision field value literally: "Decision is `approve`."
1771    #[serial(cooldown_state)]
1772    #[test]
1773    fn infer_review_response_decision_is_approve_backtick() {
1774        let text = "Already retrieved the output and completed the review above. The CI test check passed (3m11s) and all other checks are green. Decision is `approve`.";
1775        let resp = parse_review_plain(text).unwrap();
1776        assert_eq!(resp.decision, "approve");
1777    }
1778
1779    /// "Decision: approve" without backticks should also match.
1780    #[serial(cooldown_state)]
1781    #[test]
1782    fn infer_review_response_decision_colon_approve() {
1783        let text = "Reviewed the PR. No blocking issues found. Decision: approve";
1784        let resp = parse_review_plain(text).unwrap();
1785        assert_eq!(resp.decision, "approve");
1786    }
1787
1788    /// Bare backtick `approve` in a negative context must NOT infer approval.
1789    /// e.g. "do not `approve`" previously matched the removed `lower.contains("`approve`")` check.
1790    #[serial(cooldown_state)]
1791    #[test]
1792    fn infer_review_response_negation_do_not_approve_backtick() {
1793        let text = "There are unresolved issues — do not `approve` this PR until they are fixed.";
1794        assert!(
1795            parse_review_plain(text).is_err(),
1796            "negative context with backtick approve must not infer approval"
1797        );
1798    }
1799
1800    // ── review parsing via find_agent_result + parse_review_response ────
1801
1802    /// Helper that mirrors the production path in review.rs:
1803    /// find_agent_result → parse_review_response → infer_review_response.
1804    fn parse_via_agent_path(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
1805        let text = find_agent_result(agent, output)
1806            .map(|r| r.result_text)
1807            .filter(|t| !t.is_empty())
1808            .unwrap_or_else(|| output.to_string());
1809        if let Ok(r) = parse_review_response(&text) {
1810            return Ok(r);
1811        }
1812        infer_review_response_from_text(&text)
1813            .ok_or_else(|| anyhow::anyhow!("failed to parse review response from {agent} output"))
1814    }
1815
1816    #[serial(cooldown_state)]
1817    #[test]
1818    fn agent_output_claude_ndjson_review() {
1819        let ndjson = concat!(
1820            r#"{"type":"system","subtype":"init","session_id":"s1"}"#,
1821            "\n",
1822            r#"{"type":"assistant","message":{"content":[{"type":"text","text":"Reviewing..."}]}}"#,
1823            "\n",
1824            r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"test_results\":\"pass\",\"issues\":[]}","usage":{"input_tokens":100,"output_tokens":20}}"#,
1825        );
1826        let resp = parse_via_agent_path("claude", ndjson).unwrap();
1827        assert_eq!(resp.decision, "approve");
1828    }
1829
1830    #[serial(cooldown_state)]
1831    #[test]
1832    fn agent_output_opencode_ndjson_review() {
1833        let ndjson = concat!(
1834            r#"{"type":"step_start","timestamp":1000}"#,
1835            "\n",
1836            r#"{"type":"text","timestamp":1001,"part":{"type":"text","text":"{\"decision\":\"request_changes\",\"notes\":\"Fix it\",\"issues\":[]}"}}"#,
1837            "\n",
1838            r#"{"type":"step_finish","timestamp":1002,"part":{"type":"step-finish","reason":"stop","tokens":{"input":100,"output":10}}}"#,
1839        );
1840        let resp = parse_via_agent_path("opencode", ndjson).unwrap();
1841        assert_eq!(resp.decision, "request_changes");
1842    }
1843
1844    #[serial(cooldown_state)]
1845    #[test]
1846    fn agent_output_codex_ndjson_review() {
1847        let ndjson = concat!(
1848            r#"{"type":"thread.started","thread_id":"t1"}"#,
1849            "\n",
1850            r#"{"type":"item.completed","item":{"type":"agent_message","text":"{\"decision\":\"approve\",\"notes\":\"All good\",\"issues\":[]}"}}"#,
1851            "\n",
1852            r#"{"type":"turn.completed"}"#,
1853        );
1854        let resp = parse_via_agent_path("codex", ndjson).unwrap();
1855        assert_eq!(resp.decision, "approve");
1856    }
1857
1858    #[serial(cooldown_state)]
1859    #[test]
1860    fn agent_output_plain_text_keyword_fallback() {
1861        let text = "All checks passed, LGTM.";
1862        let resp = parse_via_agent_path("claude", text).unwrap();
1863        assert_eq!(resp.decision, "approve");
1864    }
1865
1866    #[serial(cooldown_state)]
1867    #[test]
1868    fn agent_output_direct_json() {
1869        let json = r#"{"decision":"approve","notes":"LGTM","test_results":"pass","issues":[]}"#;
1870        let resp = parse_via_agent_path("opencode", json).unwrap();
1871        assert_eq!(resp.decision, "approve");
1872    }
1873
1874    /// Regression: plain text containing JSON fragments should NOT trigger false
1875    /// positives. The per-agent extractor returns None for non-NDJSON text,
1876    /// so we fall through to keyword inference instead of generic NDJSON parsing.
1877    #[serial(cooldown_state)]
1878    #[test]
1879    fn agent_output_text_with_json_fragment_no_false_positive() {
1880        let text = r#"I checked the file and found {"type":"text"} in the output. No issues."#;
1881        // This should NOT extract a review from the JSON fragment.
1882        // With the generic parser, this would match as a "text" NDJSON event.
1883        // With per-agent extraction, it returns None (not valid agent NDJSON).
1884        let result = parse_via_agent_path("claude", text);
1885        // Should either fail or infer from keywords, NOT parse the JSON fragment
1886        assert!(
1887            result.is_err() || result.as_ref().is_ok_and(|r| r.decision != "text"),
1888            "should not extract review from embedded JSON fragment"
1889        );
1890    }
1891
1892    /// Prose containing the word "decision" and a JSON-like fragment must not
1893    /// be misidentified as a review response by `extract_review_response_object`.
1894    #[serial(cooldown_state)]
1895    #[test]
1896    fn parse_review_response_ignores_prose_with_decision_word() {
1897        let text = r#"The decision was made. Here is a debug trace: {"trace":123}"#;
1898        assert!(
1899            parse_review_response(text).is_err(),
1900            "prose with 'decision' word and unrelated JSON should not parse as ReviewResponse"
1901        );
1902    }
1903
1904    /// Plain text with an embedded JSON object that has unrelated keys should
1905    /// not be extracted as a review response.
1906    #[serial(cooldown_state)]
1907    #[test]
1908    fn parse_review_response_ignores_embedded_json_without_decision_key() {
1909        let text = r#"Analysis complete. Config: {"timeout":30,"retries":3}. Done."#;
1910        assert!(
1911            parse_review_response(text).is_err(),
1912            "embedded JSON without 'decision' key should not parse as ReviewResponse"
1913        );
1914    }
1915
1916    // ── read_file_with_limit tests ───────────────────────────────────────────
1917
1918    #[serial(cooldown_state)]
1919    #[test]
1920    fn read_file_with_limit_reads_small_file_completely() {
1921        let temp_dir = env::temp_dir();
1922        let file_path = temp_dir.join(format!("orch_test_small_{}.txt", process::id()));
1923        let content = "Hello, World!";
1924        fs::write(&file_path, content).unwrap();
1925
1926        let result = read_file_with_limit(&file_path, 1024);
1927        assert!(result.is_ok());
1928        assert_eq!(result.unwrap(), content);
1929
1930        let _ = fs::remove_file(&file_path);
1931    }
1932
1933    #[serial(cooldown_state)]
1934    #[test]
1935    fn read_file_with_limit_truncates_large_file() {
1936        let temp_dir = env::temp_dir();
1937        let file_path = temp_dir.join(format!("orch_test_large_{}.txt", process::id()));
1938
1939        // Create a file larger than the limit (100 bytes)
1940        let large_content = "x".repeat(200);
1941        fs::write(&file_path, &large_content).unwrap();
1942
1943        // Read with a 50-byte limit
1944        let result = read_file_with_limit(&file_path, 50);
1945        assert!(result.is_ok());
1946
1947        let read_content = result.unwrap();
1948        // Should only read 50 bytes (the tail of the file)
1949        assert_eq!(read_content.len(), 50);
1950        assert_eq!(read_content, "x".repeat(50));
1951
1952        let _ = fs::remove_file(&file_path);
1953    }
1954
1955    #[serial(cooldown_state)]
1956    #[test]
1957    fn read_file_with_limit_handles_empty_file() {
1958        let temp_dir = env::temp_dir();
1959        let file_path = temp_dir.join(format!("orch_test_empty_{}.txt", process::id()));
1960        fs::write(&file_path, "").unwrap();
1961
1962        let result = read_file_with_limit(&file_path, 1024);
1963        assert!(result.is_ok());
1964        assert_eq!(result.unwrap(), "");
1965
1966        let _ = fs::remove_file(&file_path);
1967    }
1968
1969    #[serial(cooldown_state)]
1970    #[test]
1971    fn read_file_with_limit_handles_nonexistent_file() {
1972        let temp_dir = env::temp_dir();
1973        let file_path = temp_dir.join("orch_test_nonexistent_xyz123.txt");
1974
1975        let result = read_file_with_limit(&file_path, 1024);
1976        assert!(result.is_err());
1977    }
1978
1979    #[serial(cooldown_state)]
1980    #[test]
1981    fn read_file_with_limit_reads_exact_limit() {
1982        let temp_dir = env::temp_dir();
1983        let file_path = temp_dir.join(format!("orch_test_exact_{}.txt", process::id()));
1984
1985        // Create exactly 100 bytes
1986        let content = "y".repeat(100);
1987        fs::write(&file_path, &content).unwrap();
1988
1989        // Read with exactly 100-byte limit (should read all)
1990        let result = read_file_with_limit(&file_path, 100);
1991        assert!(result.is_ok());
1992        assert_eq!(result.unwrap().len(), 100);
1993
1994        let _ = fs::remove_file(&file_path);
1995    }
1996
1997    #[serial(cooldown_state)]
1998    #[test]
1999    fn read_file_with_limit_handles_unicode_truncation() {
2000        let temp_dir = env::temp_dir();
2001        let file_path = temp_dir.join(format!("orch_test_unicode_{}.txt", process::id()));
2002
2003        // Create content with multi-byte UTF-8 characters
2004        let emoji = "🎉".repeat(100); // Each emoji is 4 bytes
2005        fs::write(&file_path, &emoji).unwrap();
2006
2007        // Read with a limit that cuts into the emoji sequence
2008        let result = read_file_with_limit(&file_path, 50);
2009        assert!(result.is_ok());
2010
2011        let read_content = result.unwrap();
2012        // from_utf8_lossy should handle invalid sequences
2013        assert_ne!(read_content, "");
2014
2015        let _ = fs::remove_file(&file_path);
2016    }
2017
2018    // ── Fixture-based review integration tests ──────────────────────────────────
2019
2020    /// Full review pipeline: find_agent_result → parse_review_response → infer_review_response.
2021    fn parse_review_via_agent(agent: &str, output: &str) -> anyhow::Result<ReviewResponse> {
2022        let text = find_agent_result(agent, output)
2023            .map(|r| r.result_text)
2024            .filter(|t| !t.is_empty())
2025            .unwrap_or_else(|| output.to_string());
2026        if let Ok(r) = parse_review_response(&text) {
2027            return Ok(r);
2028        }
2029        infer_review_response(&text)
2030            .ok_or_else(|| anyhow::anyhow!("failed to parse review from {agent} output"))
2031    }
2032
2033    // ── Claude review fixtures ─────────────────────────────────────────────────
2034
2035    #[serial(cooldown_state)]
2036    #[test]
2037    fn fixture_claude_review_approve() {
2038        let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2039        let resp = parse_review_via_agent("claude", raw).unwrap();
2040        assert_eq!(resp.decision, "approve");
2041        assert_eq!(resp.notes, "All checks pass");
2042        assert_eq!(resp.test_results.as_deref(), Some("pass"));
2043        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2044    }
2045
2046    #[serial(cooldown_state)]
2047    #[test]
2048    fn fixture_claude_review_request_changes() {
2049        let raw = include_str!("../../../tests/fixtures/review_claude_request_changes.jsonl");
2050        let resp = parse_review_via_agent("claude", raw).unwrap();
2051        assert_eq!(resp.decision, "request_changes");
2052        assert_eq!(resp.notes, "Fix the null deref on line 42");
2053        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2054        assert_eq!(resp.issues.len(), 1);
2055        assert_eq!(resp.issues[0].file, "src/main.rs");
2056        assert_eq!(resp.issues[0].line, Some(42));
2057        assert_eq!(resp.issues[0].severity, "error");
2058    }
2059
2060    #[serial(cooldown_state)]
2061    #[test]
2062    fn fixture_claude_review_rate_limit_returns_error() {
2063        // find_claude_result on an is_error=true result returns Some with is_error=true.
2064        // The integration test verifies this is captured, not that it's a ReviewResponse.
2065        let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2066        let result = find_agent_result("claude", raw);
2067        assert!(
2068            result.is_some(),
2069            "should find result even for error envelope"
2070        );
2071        let result = result.unwrap();
2072        assert!(result.is_error, "rate limit should be flagged as error");
2073        assert!(result.result_text.contains("rate limit"));
2074    }
2075
2076    // ── OpenCode review fixtures ────────────────────────────────────────────────
2077
2078    #[serial(cooldown_state)]
2079    #[test]
2080    fn fixture_opencode_review_approve() {
2081        let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2082        let resp = parse_review_via_agent("opencode", raw).unwrap();
2083        assert_eq!(resp.decision, "approve");
2084        assert_eq!(resp.notes, "All checks pass");
2085        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2086    }
2087
2088    #[serial(cooldown_state)]
2089    #[test]
2090    fn fixture_opencode_review_request_changes() {
2091        let raw = include_str!("../../../tests/fixtures/review_opencode_request_changes.jsonl");
2092        let resp = parse_review_via_agent("opencode", raw).unwrap();
2093        assert_eq!(resp.decision, "request_changes");
2094        assert_eq!(resp.notes, "Fix the memory leak");
2095        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2096        assert_eq!(resp.issues.len(), 1);
2097        assert_eq!(resp.issues[0].file, "src/lib.rs");
2098        assert_eq!(resp.issues[0].line, Some(100));
2099    }
2100
2101    #[serial(cooldown_state)]
2102    #[test]
2103    fn fixture_opencode_review_plain_text_approve() {
2104        // OpenCode with plain-text review (no JSON) — should infer approval via keywords.
2105        let raw = include_str!("../../../tests/fixtures/review_opencode_plain_text.jsonl");
2106        let resp = parse_review_via_agent("opencode", raw).unwrap();
2107        assert_eq!(resp.decision, "approve");
2108    }
2109
2110    // ── Codex review fixtures ────────────────────────────────────────────────────
2111
2112    #[serial(cooldown_state)]
2113    #[test]
2114    fn fixture_codex_review_approve() {
2115        let raw = include_str!("../../../tests/fixtures/review_codex_approve.jsonl");
2116        let resp = parse_review_via_agent("codex", raw).unwrap();
2117        assert_eq!(resp.decision, "approve");
2118        assert_eq!(resp.notes, "Clean implementation");
2119        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2120    }
2121
2122    #[serial(cooldown_state)]
2123    #[test]
2124    fn fixture_codex_review_request_changes() {
2125        let raw = include_str!("../../../tests/fixtures/review_codex_request_changes.jsonl");
2126        let resp = parse_review_via_agent("codex", raw).unwrap();
2127        assert_eq!(resp.decision, "request_changes");
2128        assert_eq!(resp.notes, "Race condition on line 15");
2129        assert_eq!(resp.issues.len(), 1);
2130        assert_eq!(resp.issues[0].file, "src/server.rs");
2131        assert_eq!(resp.issues[0].line, Some(15));
2132    }
2133
2134    #[serial(cooldown_state)]
2135    #[test]
2136    fn fixture_codex_review_plain_text_approve() {
2137        // Codex with plain-text review — should infer approval via "LGTM" keyword.
2138        let raw = include_str!("../../../tests/fixtures/review_codex_plain_text.jsonl");
2139        let resp = parse_review_via_agent("codex", raw).unwrap();
2140        assert_eq!(resp.decision, "approve");
2141    }
2142
2143    // ── Kimi review fixtures ────────────────────────────────────────────────────
2144
2145    #[serial(cooldown_state)]
2146    #[test]
2147    fn fixture_kimi_review_approve() {
2148        let raw = include_str!("../../../tests/fixtures/review_kimi_approve.jsonl");
2149        let resp = parse_review_via_agent("kimi", raw).unwrap();
2150        assert_eq!(resp.decision, "approve");
2151        assert_eq!(resp.notes, "LGTM");
2152        assert_eq!(resp.test_results.as_deref(), Some("pass"));
2153        assert!(resp.issues.is_empty(), "{:?}", resp.issues);
2154    }
2155
2156    // ── MiniMax review fixtures ────────────────────────────────────────────────
2157
2158    #[serial(cooldown_state)]
2159    #[test]
2160    fn fixture_minimax_review_request_changes() {
2161        let raw = include_str!("../../../tests/fixtures/review_minimax_request_changes.jsonl");
2162        let resp = parse_review_via_agent("minimax", raw).unwrap();
2163        assert_eq!(resp.decision, "request_changes");
2164        assert_eq!(resp.notes, "Fix the bug");
2165        assert_eq!(resp.test_results.as_deref(), Some("fail"));
2166        assert_eq!(resp.issues.len(), 1);
2167        assert_eq!(resp.issues[0].file, "src/foo.rs");
2168        assert_eq!(resp.issues[0].line, Some(42));
2169        assert_eq!(resp.issues[0].severity, "error");
2170    }
2171
2172    // ── Cross-agent token/metadata extraction ───────────────────────────────────
2173
2174    /// Verify that find_agent_result extracts token counts from each agent's format.
2175    #[serial(cooldown_state)]
2176    #[test]
2177    fn find_agent_result_extracts_tokens_claude() {
2178        let raw = include_str!("../../../tests/fixtures/review_claude_approve.jsonl");
2179        let result = find_agent_result("claude", raw).expect("should find result");
2180        assert_eq!(result.input_tokens, Some(200));
2181        assert_eq!(result.output_tokens, Some(30));
2182        assert!(result.cost_usd.is_none()); // review fixtures don't include cost
2183    }
2184
2185    #[serial(cooldown_state)]
2186    #[test]
2187    fn find_agent_result_extracts_tokens_opencode() {
2188        let raw = include_str!("../../../tests/fixtures/review_opencode_approve.jsonl");
2189        let result = find_agent_result("opencode", raw).expect("should find result");
2190        assert_eq!(result.input_tokens, Some(2900));
2191        assert_eq!(result.output_tokens, Some(100));
2192        assert_eq!(result.cost_usd, Some(0.002));
2193    }
2194
2195    /// Verify that find_agent_result returns None for empty/unparseable input.
2196    #[serial(cooldown_state)]
2197    #[test]
2198    fn find_agent_result_returns_none_for_empty() {
2199        assert!(find_agent_result("claude", "").is_none());
2200        assert!(find_agent_result("opencode", "").is_none());
2201        assert!(find_agent_result("codex", "").is_none());
2202        assert!(find_agent_result("kimi", "").is_none());
2203        assert!(find_agent_result("minimax", "").is_none());
2204    }
2205
2206    #[serial(cooldown_state)]
2207    #[test]
2208    fn find_agent_result_returns_none_for_plain_text() {
2209        assert!(find_agent_result("claude", "just some plain text").is_none());
2210        assert!(find_agent_result("codex", "just some plain text").is_none());
2211        // OpenCode is a special case — it can sometimes return a result for plain text.
2212        // This is acceptable because opencode's extract_ndjson_text falls back gracefully.
2213    }
2214
2215    // ── Error propagation via find_agent_result ─────────────────────────────────
2216
2217    /// Verify that find_agent_result surfaces rate-limit errors from Claude.
2218    #[serial(cooldown_state)]
2219    #[test]
2220    fn find_agent_result_propagates_claude_rate_limit() {
2221        let raw = include_str!("../../../tests/fixtures/review_claude_rate_limit.jsonl");
2222        let result = find_agent_result("claude", raw).expect("should find result");
2223        assert!(result.is_error, "should be flagged as error");
2224        assert!(result.result_text.contains("rate limit"));
2225    }
2226
2227    /// Two fake agents on PATH, a config that names them and a 1 ms backoff. Each nextest test
2228    /// runs in its own process, so the environment is not restored
2229    #[cfg(unix)]
2230    fn failover_env() -> tempfile::TempDir {
2231        let dir = tempfile::tempdir().unwrap();
2232        let bin = dir.path().join("bin");
2233        fs::create_dir(&bin).unwrap();
2234        for agent in ["claude", "codex"] {
2235            let path = bin.join(agent);
2236            fs::write(&path, "#!/bin/sh\n").unwrap();
2237            fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap();
2238        }
2239        let home = dir.path().join("home");
2240        fs::create_dir(&home).unwrap();
2241        fs::write(
2242            home.join("config.yml"),
2243            "agents: [claude, codex]\ndispatch:\n  retry_base_delay_ms: 1\n  retry_max_delay_ms: 1\nworkflow:\n  max_timeout_retries: 3\n",
2244        )
2245        .unwrap();
2246        let mut path = bin.as_os_str().to_os_string();
2247        path.push(":");
2248        path.push(env::var_os("PATH").unwrap_or_default());
2249        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2250        unsafe { env::set_var("PATH", path) };
2251        // SAFETY: same as above
2252        unsafe { env::set_var("ORCH_HOME", &home) };
2253        cmd_cache::reset_for_test();
2254        dir
2255    }
2256
2257    async fn failover_task() -> (Arc<TaskStore>, String) {
2258        let store = Arc::new(TaskStore::open_memory().await.expect("in-memory store"));
2259        let id = store
2260            .create_internal("owner/repo", "t", "b", "test", "", None)
2261            .await
2262            .unwrap();
2263        (store, format!("internal:{id}"))
2264    }
2265
2266    /// Every other agent is in cooldown, so the timed-out task retries from `new` after the backoff
2267    #[cfg(unix)]
2268    #[serial(cooldown_state)]
2269    #[tokio::test]
2270    async fn timeout_with_every_other_agent_cooled_retries_as_new() {
2271        let _env = failover_env();
2272        cooldown::reset_global_state().await;
2273        set_agent_cooldown("codex", 600).await;
2274        let (store, task_id) = failover_task().await;
2275        let store = Some(store);
2276
2277        let outcome = handle_failover(
2278            &task_id,
2279            "claude",
2280            RetryableError::Timeout,
2281            "claude timed out after 1804s",
2282            false,
2283            &store,
2284            "owner/repo",
2285        )
2286        .await;
2287
2288        assert_eq!(outcome.status, "new");
2289        assert!(outcome.next_agent.is_none());
2290        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2291            .await
2292            .unwrap();
2293        assert!(task.limit_reroute_chain.is_empty(), "chain must be cleared");
2294        assert!(
2295            task.last_error.contains("will retry"),
2296            "{}",
2297            task.last_error
2298        );
2299        assert_eq!(task.timeout_retries, 1);
2300    }
2301
2302    /// Another agent is free, so the timeout still reroutes to it
2303    #[cfg(unix)]
2304    #[serial(cooldown_state)]
2305    #[tokio::test]
2306    async fn timeout_with_a_free_agent_reroutes_to_it() {
2307        let _env = failover_env();
2308        cooldown::reset_global_state().await;
2309        let (store, task_id) = failover_task().await;
2310        let store = Some(store);
2311
2312        let outcome = handle_failover(
2313            &task_id,
2314            "claude",
2315            RetryableError::Timeout,
2316            "claude timed out after 1804s",
2317            false,
2318            &store,
2319            "owner/repo",
2320        )
2321        .await;
2322
2323        assert_eq!(outcome.status, "new");
2324        assert_eq!(outcome.next_agent.as_deref(), Some("codex"));
2325        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2326            .await
2327            .unwrap();
2328        assert_eq!(task.limit_reroute_chain, "claude");
2329        assert_eq!(task.agent.as_deref(), Some("codex"));
2330        assert!(
2331            task.model.as_deref().unwrap_or("").is_empty(),
2332            "model must be cleared so routing picks one for the fallback, got {:?}",
2333            task.model
2334        );
2335        assert!(
2336            task.last_error.contains("switching to codex"),
2337            "{}",
2338            task.last_error
2339        );
2340    }
2341
2342    /// The run_with_context reroute clear must keep the fallback while status is still InProgress
2343    #[cfg(unix)]
2344    #[serial(cooldown_state)]
2345    #[tokio::test]
2346    async fn timeout_failover_keeps_fallback_after_reroute_clear() {
2347        let _env = failover_env();
2348        cooldown::reset_global_state().await;
2349        let (store_arc, task_id) = failover_task().await;
2350        let store_id = store_arc
2351            .resolve_task_id("owner/repo", &task_id)
2352            .await
2353            .unwrap()
2354            .unwrap();
2355        store_arc
2356            .update_status(store_id, TaskStatus::InProgress)
2357            .await
2358            .unwrap();
2359        store_arc
2360            .set_fields(store_id, &[("agent", serde_json::json!("claude"))])
2361            .await
2362            .unwrap();
2363
2364        let store = Some(Arc::clone(&store_arc));
2365        let outcome = handle_failover(
2366            &task_id,
2367            "claude",
2368            RetryableError::Timeout,
2369            "claude timed out after 1804s",
2370            false,
2371            &store,
2372            "owner/repo",
2373        )
2374        .await;
2375        assert_eq!(outcome.status, "new");
2376        assert_eq!(outcome.next_agent.as_deref(), Some("codex"));
2377
2378        let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store_arc));
2379        runner
2380            .clear_agent_on_reroute_unless_fallback(&task_id, "claude")
2381            .await;
2382
2383        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2384            .await
2385            .unwrap();
2386        assert_eq!(task.agent.as_deref(), Some("codex"));
2387        assert_eq!(task.status, TaskStatus::InProgress);
2388        assert!(
2389            task.model.as_deref().unwrap_or("").is_empty(),
2390            "model must stay empty so routing picks one for the fallback, got {:?}",
2391            task.model
2392        );
2393
2394        let mut config = RouterConfig::default();
2395        let medium = config.model_map.entry("medium".to_string()).or_default();
2396        medium.insert("claude".to_string(), vec!["haiku".to_string()]);
2397        medium.insert("codex".to_string(), vec!["gpt-test".to_string()]);
2398        let agents = vec!["claude".to_string(), "codex".to_string()];
2399        let mut router = Router::new_for_test(config, agents);
2400        let labeled = ExternalTask {
2401            id: ExternalId(task_id.clone()),
2402            title: "Test".to_string(),
2403            body: "Test body".to_string(),
2404            state: "open".to_string(),
2405            labels: vec!["agent:claude".to_string()],
2406            author: "test".to_string(),
2407            created_at: "2026-01-01T00:00:00Z".to_string(),
2408            updated_at: "2026-01-01T00:00:00Z".to_string(),
2409            url: format!("https://github.com/owner/repo/issues/{task_id}"),
2410        };
2411        let result = router
2412            .route(&labeled, &store_arc, "owner/repo")
2413            .await
2414            .unwrap();
2415        assert_eq!(result.agent, "codex");
2416    }
2417
2418    /// A task that times out on every run blocks after `max_timeout_retries` and keeps its issue open
2419    #[cfg(unix)]
2420    #[serial(cooldown_state)]
2421    #[tokio::test]
2422    async fn repeated_timeouts_block_the_task() {
2423        let _env = failover_env();
2424        cooldown::reset_global_state().await;
2425        set_agent_cooldown("codex", 600).await;
2426        let (store, task_id) = failover_task().await;
2427        let store = Some(store);
2428
2429        let mut statuses = Vec::new();
2430        for _ in 0..3 {
2431            statuses.push(
2432                handle_failover(
2433                    &task_id,
2434                    "claude",
2435                    RetryableError::Timeout,
2436                    "claude timed out after 1804s",
2437                    false,
2438                    &store,
2439                    "owner/repo",
2440                )
2441                .await
2442                .status,
2443            );
2444        }
2445
2446        assert_eq!(statuses, ["new", "new", "blocked"]);
2447        let task = store::opt_store_get_task(&store, "owner/repo", &task_id)
2448            .await
2449            .unwrap();
2450        assert_eq!(task.block_reason.as_deref(), Some("timed out 3 times"));
2451        assert_eq!(task.timeout_retries, 3);
2452    }
2453}