Skip to main content

orch/engine/
review.rs

1//! PR review pipeline — review agent orchestration.
2//!
3//! This module handles running the review agent on completed tasks, parsing its
4//! decision, and posting automated review comments.
5//!
6//! Related modules:
7//! - [`super::review_poll`] — polling open PRs for human review feedback
8//! - [`super::auto_merge`] — merging approved PRs and handling change requests
9
10/// Maximum consecutive review agent failures before the task is blocked
11/// for human intervention. Exported so `tick` and `sync` use the same threshold.
12pub(crate) const MAX_REVIEW_AGENT_FAILURES: u64 = 3;
13
14/// Maximum consecutive PR-creation failures before blocking the task.
15const MAX_PR_CREATE_FAILURES: u64 = 3;
16
17/// Map an [`AgentError`] to the structured outcome string stored in `task_runs.outcome`.
18///
19/// Mirrors the logic in `runner::classify_run_outcome` so review runs carry the
20/// same outcome fidelity as agent runs.
21fn outcome_for_agent_error(err: &runner::agents::AgentError) -> &'static str {
22    match err {
23        runner::agents::AgentError::Timeout { .. } => "timeout",
24        // Consult credit-exhaustion detection first — generic rate-limit patterns
25        // (e.g. "usage limit") also match billing-cycle exhaustion messages, so
26        // reviews on a billing-cycle-exhausted model must not be stored as a
27        // plain "rate_limit"/"auth_error" outcome (issue #3529).
28        runner::agents::AgentError::RateLimit { message } => {
29            crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
30                .unwrap_or("rate_limit")
31        }
32        runner::agents::AgentError::Auth { message } => {
33            crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
34                .unwrap_or("auth_error")
35        }
36        runner::agents::AgentError::InvalidResponse { .. } => "parse_error",
37        _ => "failed",
38    }
39}
40
41/// Format an [`AgentError`] as the compact string persisted to `task_runs.error`
42/// and surfaced via [`ReviewDecision::Failed`].
43///
44/// Mirrors the per-variant formatting in `runner::fallback::handle_error` so review
45/// runs never leak raw provider payloads (e.g. truncated `api_retry` NDJSON
46/// fragments) into the audit trail. Without this, `err.to_string()` would expand
47/// `AgentError::RateLimit { message }` via its `Display` impl and persist hundreds
48/// of bytes of raw JSON into `task_runs.error` (issue #3149, regression of #2881).
49fn format_review_error(agent: &str, err: &runner::agents::AgentError) -> String {
50    match err {
51        runner::agents::AgentError::RateLimit { message } => format!(
52            "{agent} rate limit: {}",
53            runner::fallback::summarize_rate_limit_error(message)
54        ),
55        runner::agents::AgentError::Auth { message } => format!("{agent} auth error: {message}"),
56        runner::agents::AgentError::Timeout { elapsed } => {
57            format!("{agent} timed out after {}s", elapsed.as_secs())
58        }
59        other => format!("{agent} agent error: {other}"),
60    }
61}
62
63/// Parse a PR number from a GitHub PR URL.
64///
65/// Validates that the URL matches the expected GitHub PR URL format
66/// (`https://github.com/<owner>/<repo>/pull/<number>`) before extracting the
67/// number. Returns `Err` with a descriptive message for any URL that doesn't
68/// conform to this structure, including URLs with wrong domains, missing
69/// `/pull/` segments, or non-numeric trailing components.
70///
71/// The extracted segment is stripped of any query string (`?`) or hash
72/// fragment (`#`) before parsing so that URLs like
73/// `https://github.com/owner/repo/pull/42?tab=files` are handled correctly.
74pub(crate) fn parse_pr_number_from_url(url: &str) -> Result<u64, String> {
75    let url = url.trim();
76    if url.is_empty() {
77        return Err("URL is empty".to_string());
78    }
79    if !url.starts_with("https://github.com/") {
80        return Err(format!(
81            "URL does not start with 'https://github.com/': {url}"
82        ));
83    }
84    if !url.contains("/pull/") {
85        return Err(format!("URL missing '/pull/' segment: {url}"));
86    }
87    let last_segment = url
88        .trim_end_matches('/')
89        .rsplit('/')
90        .next()
91        .ok_or_else(|| format!("No segment after '/pull/' in URL: {url}"))?;
92    let number_part = last_segment
93        .split(['?', '#'])
94        .next()
95        .filter(|s| !s.is_empty())
96        .ok_or_else(|| format!("PR number segment is empty in URL: {url}"))?;
97    let pr_num: u64 = number_part.parse().map_err(|_| {
98        format!("PR number is not a valid positive integer: '{number_part}' in URL: {url}")
99    })?;
100    if pr_num == 0 {
101        return Err(format!("PR number cannot be zero in URL: {url}"));
102    }
103    Ok(pr_num)
104}
105
106fn review_started_comment(review_agent: &str, review_model: &str) -> String {
107    let footer = attribution_footer("Review started", review_agent, Some(review_model));
108    format!("🔍 Automated review started{}", footer)
109}
110
111fn should_skip_no_code_reroute_increment(last_error: &str) -> bool {
112    crate::engine::runner::git_ops::is_transient_github_api_error(last_error)
113}
114
115use crate::backends::{ExternalBackend, ExternalTask, Status};
116use crate::cmd::CommandErrorContext;
117use crate::config;
118use crate::engine::attribution_footer;
119use crate::engine::auto_merge::{auto_merge_pr, handle_review_changes};
120use crate::engine::runner;
121use crate::engine::runner::worktree;
122use crate::engine::tasks::TaskManager;
123use crate::github::http::GhHttp;
124use crate::store::store_log_activity;
125use crate::store::TaskStore;
126use crate::store::{opt_store_get_task, set_review_session_expected, store_increment};
127use crate::store::{store_set, store_set_result};
128use crate::store::{CompleteRun, RunTokenUsage, StartRun};
129use crate::tmux::TmuxManager;
130use anyhow::Context;
131use std::collections::HashSet;
132use std::sync::Arc;
133use tokio::process::Command;
134use tokio::sync::RwLock;
135
136use super::router::Router;
137
138/// Review agent decision result.
139#[derive(Debug, Clone)]
140pub(crate) enum ReviewDecision {
141    /// Review approved, PR can be merged.
142    Approve,
143    /// Changes requested, PR needs fixes.
144    RequestChanges {
145        notes: String,
146        issues: Vec<crate::engine::runner::response::ReviewIssue>,
147    },
148    /// Review agent failed or crashed (reason stored for logging).
149    Failed(String),
150    /// Unrecoverable failure — task should be blocked for human intervention.
151    Blocked(String),
152    /// No PR exists — nothing to review, task marked done directly.
153    Skipped,
154    /// No PR and no commits — task re-routed for retry (no_code_reroutes counter must not be reset).
155    Rerouted,
156}
157
158/// Outcome of [`ensure_pr_exists`]: either a ready PR number or an early return decision.
159enum EnsurePrResult {
160    /// PR exists (found in store or via API, or just created). Proceed with review.
161    Ready(u64),
162    /// No PR and status already updated — return this decision to the caller.
163    EarlyReturn(ReviewDecision),
164}
165
166enum ReviewPhase<T> {
167    Ready(T),
168    EarlyReturn(ReviewDecision),
169}
170
171struct ReviewContext {
172    store_id: Option<i64>,
173    had_prev_push_failure: bool,
174    review_attempt: u32,
175    worktree_path: std::path::PathBuf,
176    branch_name: String,
177    pr_number: u64,
178    /// PR head SHA when the review started, the commit the decision applies to
179    head_sha: Option<String>,
180    default_branch: String,
181    review_agent: String,
182    review_model: Option<String>,
183    review_task_id: String,
184    review_attempt_dir: std::path::PathBuf,
185    output_file: std::path::PathBuf,
186    invocation: runner::agent::AgentInvocation,
187    same_agent_fallback: bool,
188}
189
190impl ReviewContext {
191    fn review_model_str(&self) -> &str {
192        self.review_model.as_deref().unwrap_or("")
193    }
194}
195
196struct ReviewRun {
197    run_id: Option<i64>,
198}
199
200struct ParsedReview {
201    run_id: Option<i64>,
202    exit_code: i32,
203    stderr: String,
204    raw_output: String,
205    text_for_review: String,
206    token_usage: RunTokenUsage,
207    review_notes_for_comment: String,
208    decision: ReviewDecision,
209}
210
211fn build_review_disallowed_tools(worktree_path: &std::path::Path) -> Vec<String> {
212    let mut tools = vec![
213        "Bash(rm *)".to_string(),
214        "Bash(rm -*)".to_string(),
215        "Bash(git push*)".to_string(),
216    ];
217    let orch_yml = worktree_path.join(".orch.yml");
218    let orch_yml_str = orch_yml.to_string_lossy();
219    tools.extend([
220        format!("Write({orch_yml_str})"),
221        format!("Edit({orch_yml_str})"),
222    ]);
223    if let Ok(orch_home) = crate::home::orch_home() {
224        for path in [
225            orch_home.join("config.yml"),
226            orch_home.join("config.example.yml"),
227        ] {
228            let path_str = path.to_string_lossy();
229            tools.extend([
230                format!("Read({path_str})"),
231                format!("Write({path_str})"),
232                format!("Edit({path_str})"),
233            ]);
234        }
235    }
236    tools
237}
238
239async fn select_review_agent(
240    task_id: &str,
241    task_agent: &str,
242    store_id: Option<i64>,
243    router: &Arc<RwLock<Router>>,
244    store: &Arc<TaskStore>,
245) -> anyhow::Result<(String, Option<String>, bool)> {
246    let mut exclude_set: HashSet<String> = HashSet::new();
247    if !task_agent.is_empty() {
248        exclude_set.insert(task_agent.to_string());
249    }
250    if let Some(store_id) = store_id {
251        if let Ok(failed) = store.previous_review_agents(store_id).await {
252            for agent in failed {
253                exclude_set.insert(agent);
254            }
255        }
256    }
257
258    let mut r = router.write().await;
259    let mut tried_agents: HashSet<String> = HashSet::new();
260    let mut chosen_agent: Option<String> = None;
261    let mut chosen_model: Option<String> = None;
262    let available_count = r.available_agents.len().max(1);
263
264    loop {
265        let tmp_exclude_refs: Vec<&str> = exclude_set.iter().map(|s| s.as_str()).collect();
266        let agent = match r.next_round_robin_agent(&tmp_exclude_refs, "review") {
267            Some(a) => a,
268            None => break,
269        };
270
271        if tried_agents.contains(&agent) {
272            break;
273        }
274        tried_agents.insert(agent.clone());
275
276        let model = r.config.model_for_complexity(&agent, "review", task_id);
277        if model.is_some() {
278            chosen_agent = Some(agent);
279            chosen_model = model;
280            break;
281        }
282
283        exclude_set.insert(agent);
284        if exclude_set.len() >= available_count {
285            break;
286        }
287    }
288
289    if let Some(agent) = chosen_agent {
290        let same = !task_agent.is_empty() && agent == task_agent;
291        Ok((agent, chosen_model, same))
292    } else {
293        let final_exclude_refs: Vec<&str> = exclude_set.iter().map(|s| s.as_str()).collect();
294        let fallback_agent = r
295            .next_round_robin_agent(&final_exclude_refs, "review")
296            .unwrap_or_else(|| "claude".to_string());
297        let fallback_model = r
298            .config
299            .model_for_complexity(&fallback_agent, "review", task_id);
300        if fallback_model.is_none() {
301            anyhow::bail!(
302                "no review agent with a configured review model is available for task {}",
303                task_id
304            );
305        }
306        let same = !task_agent.is_empty() && fallback_agent == task_agent;
307        Ok((fallback_agent, fallback_model, same))
308    }
309}
310
311#[allow(clippy::too_many_arguments)]
312async fn build_review_context(
313    task: &ExternalTask,
314    repo: &str,
315    router: &Arc<RwLock<Router>>,
316    task_manager: &Arc<TaskManager>,
317    store: &Arc<TaskStore>,
318) -> anyhow::Result<ReviewPhase<ReviewContext>> {
319    let stored_task = store
320        .get_by_external_id(repo, &task.id.0)
321        .await
322        .with_context(|| format!("store lookup failed for task {}", task.id.0))?;
323
324    let store_id: Option<i64> = stored_task.as_ref().map(|t| t.id);
325    let had_prev_push_failure = stored_task
326        .as_ref()
327        .is_some_and(|t| t.last_error.contains("push failed"));
328    let review_attempt: u32 = match store_increment(
329        &Some(Arc::clone(store)),
330        repo,
331        &task.id.0,
332        "review_invocations",
333    )
334    .await
335    {
336        Ok(v) => v as u32,
337        Err(e) => {
338            tracing::warn!(task_id = task.id.0, err = %e, "failed to increment review_invocations — using fallback attempt=1");
339            1
340        }
341    };
342
343    let no_code_review = stored_task.as_ref().is_some_and(|t| {
344        t.worktree.trim().is_empty() && t.branch.trim().is_empty() && t.pr_number.is_none()
345    });
346    if no_code_review {
347        if crate::engine::tasks::is_internal_id(&task.id.0) {
348            // Internal tasks legitimately reach needs_review with nothing to review.
349            tracing::info!(
350                task_id = task.id.0,
351                "no worktree/branch/pr for internal needs_review task — skipping review and marking done"
352            );
353            if let Err(e) = task_manager
354                .update_task_status(&task.id, Status::Done)
355                .await
356            {
357                tracing::error!(
358                    task_id = task.id.0,
359                    err = %e,
360                    "update_task_status(Done) failed for no-code review skip"
361                );
362            }
363            return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Skipped));
364        }
365
366        // External task with no code produced: re-route instead of marking done.
367        // Shares the no_code_reroutes circuit breaker with the no-PR/no-commits path below.
368        let max_reroutes: u32 = config::get("workflow.max_reroute_attempts")
369            .ok()
370            .and_then(|s| s.parse().ok())
371            .or_else(|| {
372                config::get("workflow.max_attempts")
373                    .ok()
374                    .and_then(|s| s.parse().ok())
375            })
376            .unwrap_or(3);
377
378        let reroutes = match store_increment(
379            &Some(Arc::clone(store)),
380            repo,
381            &task.id.0,
382            "no_code_reroutes",
383        )
384        .await
385        {
386            Ok(v) => v,
387            Err(e) => {
388                tracing::warn!(task_id = task.id.0, err = %e, "failed to increment no_code_reroutes — skipping reroute/block decision this tick");
389                return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!(
390                    "transient store error: {e}"
391                ))));
392            }
393        };
394
395        if reroutes as u32 >= max_reroutes {
396            tracing::error!(
397                task_id = task.id.0,
398                reroutes,
399                max_reroutes,
400                "reached max reroute attempts for no-worktree/branch/pr result — blocking for human review"
401            );
402            let msg = format!(
403                "no worktree/branch/pr after {}/{} reroute attempts",
404                reroutes, max_reroutes
405            );
406            let fields = [
407                (
408                    "block_reason",
409                    serde_json::json!(format!(
410                        "max reroute attempts ({}) reached — no worktree/branch/pr produced",
411                        max_reroutes
412                    )),
413                ),
414                ("last_error", serde_json::json!(msg)),
415                ("agent", serde_json::json!(null)),
416                ("model", serde_json::json!(null)),
417            ];
418            if let Err(e) = task_manager
419                .update_task_status_and_result(&task.id, Status::Blocked, &fields)
420                .await
421            {
422                tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
423            }
424            return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Blocked(msg)));
425        }
426
427        tracing::warn!(
428            task_id = task.id.0,
429            reroutes,
430            max_reroutes,
431            "no worktree/branch/pr for external needs_review task — re-routing to a different agent"
432        );
433        if let Err(e) = store_set_result(
434            &Some(Arc::clone(store)),
435            repo,
436            &task.id.0,
437            &[
438                ("agent", serde_json::json!(null)),
439                ("model", serde_json::json!(null)),
440                (
441                    "last_error",
442                    serde_json::json!("no worktree/branch/pr produced"),
443                ),
444            ],
445        )
446        .await
447        {
448            tracing::error!(task_id = task.id.0, err = %e, "failed to clear agent/model for reroute — skipping status transition");
449        } else if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
450            tracing::error!(task_id = task.id.0, err = %e, "update_task_status(New) failed — task may be stuck in InReview");
451        }
452        return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Rerouted));
453    }
454
455    let mut worktree_path = match stored_task.as_ref().map(|t| t.worktree.as_str()) {
456        Some(w) if !w.is_empty() => std::path::PathBuf::from(w),
457        _ => {
458            tracing::warn!(task_id = task.id.0, "no worktree found for review");
459            return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(
460                "no worktree found".to_string(),
461            )));
462        }
463    };
464
465    let mut branch_name = match stored_task.as_ref().map(|t| t.branch.as_str()) {
466        Some(b) if !b.is_empty() => b.to_string(),
467        _ => {
468            tracing::warn!(task_id = task.id.0, "no branch found for review");
469            return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Failed(
470                "no branch found".to_string(),
471            )));
472        }
473    };
474
475    let mut missing_worktree_recovered = false;
476    if !worktree_path.exists() {
477        match recover_missing_review_worktree(task, repo, store).await {
478            Ok(recovered) => {
479                branch_name = recovered.branch;
480                worktree_path = recovered.work_dir;
481                missing_worktree_recovered = true;
482            }
483            Err(e) => {
484                let reason = format!("missing review worktree recovery failed: {e}");
485                tracing::error!(task_id = task.id.0, error = %reason, "cannot recover review worktree");
486                store_set(
487                    &Some(Arc::clone(store)),
488                    repo,
489                    &task.id.0,
490                    &[("last_error", serde_json::json!(reason.clone()))],
491                )
492                .await;
493                return Ok(ReviewPhase::EarlyReturn(ReviewDecision::Blocked(reason)));
494            }
495        }
496    }
497
498    if missing_worktree_recovered {
499        store_set(
500            &Some(Arc::clone(store)),
501            repo,
502            &task.id.0,
503            &[("last_error", serde_json::json!(""))],
504        )
505        .await;
506    }
507
508    let agent_summary = stored_task
509        .as_ref()
510        .map(|t| t.summary.clone())
511        .unwrap_or_default();
512    let stored_pr_number = stored_task
513        .as_ref()
514        .and_then(|t| t.pr_number)
515        .map(|n| n as u64)
516        .filter(|&n| n > 0);
517
518    let pr_number = match ensure_pr_exists(
519        task,
520        &branch_name,
521        &worktree_path,
522        repo,
523        &agent_summary,
524        store,
525        task_manager,
526        stored_pr_number,
527    )
528    .await?
529    {
530        EnsurePrResult::Ready(n) => n,
531        EnsurePrResult::EarlyReturn(decision) => return Ok(ReviewPhase::EarlyReturn(decision)),
532    };
533
534    // Run `git fetch` with a timeout so a hung network call cannot stall the
535    // review pipeline indefinitely. Treat a timeout or non-zero exit as a
536    // non-fatal warning (stale remote refs are acceptable for the review).
537    {
538        let fetch_timeout = std::time::Duration::from_secs(60);
539        let fetch_future = async {
540            let mut cmd = Command::new("git");
541            cmd.args(["fetch", "origin", "--prune"])
542                .current_dir(&worktree_path);
543            // Ensure the child is killed if the timeout elapses
544            cmd.kill_on_drop(true);
545            cmd.output_with_context().await
546        };
547
548        match tokio::time::timeout(fetch_timeout, fetch_future).await {
549            Ok(Ok(o)) if o.status.success() => {
550                // ok
551            }
552            Ok(Ok(o)) => {
553                tracing::warn!(
554                    task_id = task.id.0,
555                    exit_code = ?o.status.code(),
556                    stdout = %String::from_utf8_lossy(&o.stdout),
557                    stderr = %String::from_utf8_lossy(&o.stderr),
558                    "review: git fetch failed (non-zero exit) — diff/log may use stale remote refs"
559                );
560            }
561            Ok(Err(e)) => {
562                tracing::warn!(
563                    task_id = task.id.0,
564                    error = %e,
565                    "review: git fetch failed — diff/log may use stale remote refs"
566                );
567            }
568            Err(_) => {
569                tracing::warn!(
570                    task_id = task.id.0,
571                    "review: git fetch timed out after 60s — diff/log may use stale remote refs"
572                );
573            }
574        }
575    }
576
577    let default_branch = runner::worktree::detect_default_branch(&worktree_path).await;
578    let git_diff = runner::context::build_git_diff(&worktree_path, &default_branch).await;
579    let git_log = runner::context::build_git_log(&worktree_path, &default_branch).await;
580    let review_prompt = runner::agent::build_review_prompt(
581        task,
582        &agent_summary,
583        &git_diff,
584        &git_log,
585        &default_branch,
586        pr_number,
587        runner::agent::Shipcheck::detect(&worktree_path).await,
588    );
589
590    let task_agent = stored_task
591        .as_ref()
592        .and_then(|t| t.agent.clone())
593        .unwrap_or_default();
594    let (review_agent, review_model, same_agent_fallback) =
595        select_review_agent(&task.id.0, &task_agent, store_id, router, store).await?;
596
597    anyhow::ensure!(
598        review_model.is_some(),
599        "review model missing for task {} after agent selection — refusing to dispatch",
600        task.id.0
601    );
602
603    if same_agent_fallback {
604        tracing::warn!(
605            task_id = task.id.0,
606            agent = %review_agent,
607            "no other agent available for review — falling back to the agent that wrote the code"
608        );
609        store_log_activity(
610            &Some(Arc::clone(store)),
611            repo,
612            &task.id.0,
613            "review_agent_fallback",
614            None,
615            None,
616            Some(&review_agent),
617            review_model.as_deref(),
618            Some(&serde_json::json!({
619                "reason": "all available agents excluded (author + previous reviewers)",
620            })),
621        )
622        .await;
623    }
624
625    tracing::info!(
626        task_id = task.id.0,
627        agent = %review_agent,
628        model = ?review_model,
629        "spawning review agent"
630    );
631
632    let review_task_id = format!("{}-review", task.id.0);
633    let review_attempt_dir =
634        crate::home::task_attempt_dir_async(repo, &review_task_id, review_attempt).await?;
635    let output_file = review_attempt_dir.join("output.json");
636
637    let git_name =
638        crate::config::get("git.name").unwrap_or_else(|_| format!("{review_agent}[bot]"));
639    let git_email = crate::config::get("git.email")
640        .unwrap_or_else(|_| format!("{review_agent}[bot]@users.noreply.github.com"));
641    let review_timeout_secs: u64 = crate::config::get("workflow.review_timeout_seconds")
642        .ok()
643        .and_then(|s| s.parse().ok())
644        .unwrap_or_else(|| {
645            crate::config::get("workflow.timeout_seconds")
646                .ok()
647                .and_then(|s| s.parse().ok())
648                .unwrap_or(1800)
649        })
650        .max(1800);
651
652    let invocation = runner::agent::AgentInvocation {
653        agent: review_agent.clone(),
654        model: review_model.clone(),
655        work_dir: worktree_path.clone(),
656        system_prompt: runner::agent::review_system_prompt(),
657        agent_message: review_prompt,
658        task_id: review_task_id.clone(),
659        disallowed_tools: build_review_disallowed_tools(&worktree_path),
660        git_author_name: git_name,
661        git_author_email: git_email,
662        output_file: output_file.clone(),
663        timeout_seconds: review_timeout_secs,
664        repo: repo.to_string(),
665        attempt: review_attempt,
666    };
667
668    let head_sha = match GhHttp::new() {
669        Ok(gh) => gh.get_pr(repo, pr_number).await.ok().map(|pr| pr.head.sha),
670        Err(_) => None,
671    };
672
673    Ok(ReviewPhase::Ready(ReviewContext {
674        store_id,
675        had_prev_push_failure,
676        review_attempt,
677        worktree_path,
678        branch_name,
679        pr_number,
680        head_sha,
681        default_branch,
682        review_agent,
683        review_model,
684        review_task_id,
685        review_attempt_dir,
686        output_file,
687        invocation,
688        same_agent_fallback,
689    }))
690}
691
692#[allow(clippy::too_many_arguments)]
693async fn complete_review_run(
694    store: &Arc<TaskStore>,
695    run_id: Option<i64>,
696    exit_code: Option<i32>,
697    stdout: &str,
698    stderr: &str,
699    parsed: &str,
700    outcome: &str,
701    error: &str,
702    tokens: RunTokenUsage,
703) {
704    if let Some(run_id) = run_id {
705        if let Err(e) = store
706            .complete_run(&CompleteRun {
707                run_id,
708                exit_code,
709                stdout,
710                stderr,
711                parsed,
712                outcome,
713                error,
714                tokens,
715            })
716            .await
717        {
718            tracing::warn!(
719                run_id,
720                error = %e,
721                "failed to record review run completion in audit trail"
722            );
723        }
724    }
725}
726
727async fn invoke_review_agent(
728    task: &ExternalTask,
729    tmux: &Arc<TmuxManager>,
730    ctx: &ReviewContext,
731    repo: &str,
732    store: &Arc<TaskStore>,
733) -> ReviewPhase<ReviewRun> {
734    let run_id = if let Some(sid) = ctx.store_id {
735        match store
736            .start_run(&StartRun {
737                task_id: sid,
738                attempt: ctx.review_attempt as i32,
739                run_type: "review",
740                agent: &ctx.review_agent,
741                model: ctx.review_model.as_deref().unwrap_or(""),
742                command: "",
743                prompt: "",
744            })
745            .await
746        {
747            Ok(run_id) => Some(run_id),
748            Err(e) => {
749                tracing::warn!(
750                    task_id = task.id.0,
751                    error = %e,
752                    "failed to record review run start in audit trail"
753                );
754                None
755            }
756        }
757    } else {
758        None
759    };
760
761    let session = match runner::agent::spawn_in_tmux(tmux, &ctx.invocation).await {
762        Ok(session) => session,
763        Err(e) => {
764            tracing::error!(task_id = task.id.0, error = %e, "failed to spawn review agent");
765            complete_review_run(
766                store,
767                run_id,
768                None,
769                "",
770                &e.to_string(),
771                "",
772                "failed",
773                &format!("spawn failed: {e}"),
774                RunTokenUsage::default(),
775            )
776            .await;
777            return ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!("spawn failed: {e}")));
778        }
779    };
780
781    let start_comment = review_started_comment(&ctx.review_agent, ctx.review_model_str());
782    match GhHttp::new() {
783        Ok(gh) => {
784            post_review_status(
785                &gh,
786                repo,
787                ctx.head_sha.as_deref(),
788                "pending",
789                "Automated review running",
790            )
791            .await;
792            if let Err(e) = gh
793                .add_comment(repo, &ctx.pr_number.to_string(), &start_comment)
794                .await
795            {
796                tracing::debug!(
797                    task_id = task.id.0,
798                    pr_number = ctx.pr_number,
799                    error = %e,
800                    "failed to post review-started comment"
801                );
802            }
803        }
804        Err(e) => {
805            tracing::debug!(
806                task_id = task.id.0,
807                pr_number = ctx.pr_number,
808                error = %e,
809                "failed to create GitHub client for review-started comment"
810            );
811        }
812    }
813
814    let poll_interval = std::time::Duration::from_secs(5);
815    let timeout_duration = std::time::Duration::from_secs(ctx.invocation.timeout_seconds);
816    let wait_result = tokio::time::timeout(
817        timeout_duration,
818        tmux.wait_for_completion(&session, poll_interval),
819    )
820    .await;
821
822    match wait_result {
823        Ok(Ok(_)) => {
824            tracing::info!(task_id = task.id.0, "review agent completed");
825            let _ = tmux.kill_session(&session).await;
826            ReviewPhase::Ready(ReviewRun { run_id })
827        }
828        Ok(Err(e)) => {
829            tracing::warn!(task_id = task.id.0, error = %e, "review agent process exited with error");
830            let _ = tmux.kill_session(&session).await;
831
832            // Before recording failure, check if this is a "successful exit with non-zero
833            // code" agent like kimi/minimax. Read the output file and check for a valid
834            // result with is_error=false — if found, treat as success and proceed to
835            // parse_review_output where the real result will be extracted.
836            let exit_code: i32 = tokio::task::spawn_blocking({
837                let review_attempt_dir = ctx.review_attempt_dir.clone();
838                move || {
839                    std::fs::read_to_string(review_attempt_dir.join("exit.txt"))
840                        .ok()
841                        .and_then(|s| s.trim().parse().ok())
842                        .unwrap_or(-1)
843                }
844            })
845            .await
846            .unwrap_or(-1);
847
848            if exit_code != 0 {
849                // Try reading output via read_output_file which checks both the primary path
850                // and all prior attempt dirs as a fallback. This handles agents like kimi/minimax
851                // that exit non-zero even on success, as well as cases where the current attempt's
852                // output.json was not written but a prior attempt's output contains a valid result.
853                let raw_output =
854                    runner::response::read_output_file(&ctx.review_task_id, &ctx.output_file, repo)
855                        .await;
856                let agent_result =
857                    runner::agents::find_agent_result(&ctx.review_agent, &raw_output);
858                // Treat as success if we found a valid result without is_error set
859                if agent_result.as_ref().map(|r| !r.is_error).unwrap_or(false) {
860                    tracing::info!(
861                        task_id = task.id.0,
862                        agent = %ctx.review_agent,
863                        exit_code,
864                        output_found = !raw_output.is_empty(),
865                        "review agent succeeded despite non-zero exit — proceeding to parse output"
866                    );
867                    return ReviewPhase::Ready(ReviewRun { run_id });
868                }
869            }
870
871            tracing::error!(task_id = task.id.0, error = %e, "review agent failed");
872            complete_review_run(
873                store,
874                run_id,
875                Some(exit_code),
876                "",
877                &e.to_string(),
878                "",
879                "failed",
880                &format!("agent error: {e}"),
881                RunTokenUsage::default(),
882            )
883            .await;
884            ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!("agent error: {e}")))
885        }
886        Err(_) => {
887            tracing::error!(task_id = task.id.0, "review agent timed out");
888            let _ = tmux.kill_session(&session).await;
889            if let Some(model) = ctx.review_model.as_deref() {
890                // Structural review failure: the model cannot finish review inside
891                // the allowed time. Use persistent backoff (4h base → 7d max) so
892                // a successful agent run on the same model does not reset the
893                // failure counter and let this model be re-selected for review.
894                crate::engine::cooldown::record_persistent_model_failure(&ctx.review_agent, model)
895                    .await;
896            }
897            complete_review_run(
898                store,
899                run_id,
900                Some(-1),
901                "",
902                "timeout",
903                "",
904                "timeout",
905                "timeout",
906                RunTokenUsage::default(),
907            )
908            .await;
909            ReviewPhase::EarlyReturn(ReviewDecision::Failed("timeout".to_string()))
910        }
911    }
912}
913
914async fn record_review_agent_failure(
915    task_id: &str,
916    review_agent: &str,
917    review_model: Option<&str>,
918    err: &runner::agents::AgentError,
919) {
920    match err {
921        runner::agents::AgentError::RateLimit { message }
922        | runner::agents::AgentError::Auth { message } => {
923            tracing::warn!(
924                task_id,
925                agent = %review_agent,
926                "review agent hit error — adding to cooldown"
927            );
928            if let Some(reason) = crate::engine::cooldown::detect_credit_exhaustion(message) {
929                if reason == crate::engine::cooldown::CreditExhaustionReason::BillingCycleExhausted
930                {
931                    if let Some(model) = review_model {
932                        // Quota is model-scoped: don't penalize the whole agent.
933                        crate::engine::cooldown::record_persistent_model_failure(
934                            review_agent,
935                            model,
936                        )
937                        .await;
938                    } else {
939                        crate::engine::cooldown::record_credit_exhaustion(review_agent, reason)
940                            .await;
941                    }
942                } else {
943                    crate::engine::cooldown::record_credit_exhaustion(review_agent, reason).await;
944                }
945            } else {
946                runner::response::record_agent_failure_with_message(review_agent, &err.to_string())
947                    .await;
948            }
949        }
950        runner::agents::AgentError::InvalidResponse { .. } => {
951            // Structural failure: model produced no parseable output at all.
952            // Apply persistent backoff (4h base → 7d max) — same rationale as
953            // silent exit-0 in the agent runner. A model that cannot produce
954            // ANY output for a review is unlikely to self-heal on the next try.
955            if let Some(model) = review_model {
956                crate::engine::cooldown::record_persistent_model_failure(review_agent, model).await;
957            }
958        }
959        runner::agents::AgentError::ModelUnavailable { .. } => {
960            // Model not available (provider error, model not found, etc.)
961            // Apply persistent model cooldown (4h base → 7d max) per settled
962            // architecture (AGENTS.md:325-328). The agent itself is not
963            // penalized — its other models still route normally.
964            if let Some(model) = review_model {
965                crate::engine::cooldown::record_persistent_model_failure(review_agent, model).await;
966            }
967        }
968        _ => {
969            // Generic failure (AgentFailed, NetworkError, etc.)
970            // Apply standard model backoff (5min→4h max) to prevent immediate re-selection.
971            if let Some(model) = review_model {
972                crate::engine::cooldown::record_model_failure(review_agent, model).await;
973            }
974        }
975    }
976}
977
978fn review_decision_from_response(
979    review_response: runner::response::ReviewResponse,
980) -> ReviewDecision {
981    match review_response.decision.as_str() {
982        "approve" => ReviewDecision::Approve,
983        "request_changes" => ReviewDecision::RequestChanges {
984            notes: review_response.notes,
985            issues: review_response.issues,
986        },
987        _ => ReviewDecision::Failed(format!("unknown decision: {}", review_response.decision)),
988    }
989}
990
991fn review_decision_name(decision: &ReviewDecision) -> &'static str {
992    match decision {
993        ReviewDecision::Approve => "approve",
994        ReviewDecision::RequestChanges { .. } => "request_changes",
995        ReviewDecision::Failed(_) => "failed",
996        ReviewDecision::Blocked(_) => "blocked",
997        ReviewDecision::Skipped => "skipped",
998        ReviewDecision::Rerouted => "rerouted",
999    }
1000}
1001
1002async fn parse_review_output(
1003    task: &ExternalTask,
1004    repo: &str,
1005    ctx: &ReviewContext,
1006    run: ReviewRun,
1007    store: &Arc<TaskStore>,
1008) -> ReviewPhase<ParsedReview> {
1009    let (file_exists, file_size, exit_code, stderr) = {
1010        let output_file_clone = ctx.output_file.clone();
1011        let review_attempt_dir_clone = ctx.review_attempt_dir.clone();
1012        match tokio::task::spawn_blocking(move || {
1013            let file_exists = output_file_clone.exists();
1014            let file_size = std::fs::metadata(&output_file_clone)
1015                .map(|m| m.len())
1016                .unwrap_or(0);
1017            let exit_code = std::fs::read_to_string(review_attempt_dir_clone.join("exit.txt"))
1018                .ok()
1019                .and_then(|s| s.trim().parse::<i32>().ok())
1020                .unwrap_or(-1);
1021            let stderr = std::fs::read_to_string(review_attempt_dir_clone.join("stderr.txt"))
1022                .unwrap_or_default();
1023            (file_exists, file_size, exit_code, stderr)
1024        })
1025        .await
1026        {
1027            Ok(tuple) => tuple,
1028            Err(e) => {
1029                tracing::error!(
1030                    task_id = task.id.0,
1031                    error = %e,
1032                    "spawn_blocking panicked reading review output metadata"
1033                );
1034                (false, 0, -1, format!("spawn_blocking failed: {e}"))
1035            }
1036        }
1037    };
1038
1039    tracing::info!(
1040        task_id = task.id.0,
1041        path = %ctx.output_file.display(),
1042        file_exists,
1043        file_size,
1044        "reading review agent output"
1045    );
1046
1047    let raw_output =
1048        runner::response::read_output_file(&ctx.review_task_id, &ctx.output_file, repo).await;
1049    let raw_output = runner::agents::claude::strip_tty_control_lines(&raw_output);
1050    let has_result_envelope =
1051        runner::agents::claude::find_ndjson_result_line(&raw_output).is_some();
1052    let agent_result = runner::agents::find_agent_result(&ctx.review_agent, &raw_output);
1053    let token_usage = RunTokenUsage {
1054        input_tokens: agent_result
1055            .as_ref()
1056            .and_then(|r| r.input_tokens)
1057            .unwrap_or(0),
1058        output_tokens: agent_result
1059            .as_ref()
1060            .and_then(|r| r.output_tokens)
1061            .unwrap_or(0),
1062        total_cost_usd: agent_result
1063            .as_ref()
1064            .and_then(|r| r.cost_usd)
1065            .unwrap_or(0.0),
1066        duration_secs: 0.0,
1067    };
1068
1069    let agent_result_is_error = agent_result.as_ref().map(|r| r.is_error).unwrap_or(false);
1070    let ndjson_completed_no_error = raw_output.contains("\"terminal_reason\":\"completed\"")
1071        && !raw_output.contains("\"is_error\":true");
1072    // Do not short-circuit completed NDJSON sessions before rescue parsing.
1073    let is_hard_failure =
1074        !ndjson_completed_no_error && (raw_output.is_empty() || agent_result_is_error);
1075
1076    if is_hard_failure {
1077        tracing::warn!(
1078            task_id = task.id.0,
1079            exit_code,
1080            raw_output_len = raw_output.len(),
1081            stderr_len = stderr.len(),
1082            "review agent: entering error path"
1083        );
1084        let agent_runner = runner::agents::get_runner(&ctx.review_agent);
1085        let err = agent_runner.classify_error(exit_code, &raw_output, &stderr);
1086        record_review_agent_failure(
1087            &task.id.0,
1088            &ctx.review_agent,
1089            ctx.review_model.as_deref(),
1090            &err,
1091        )
1092        .await;
1093        tracing::error!(task_id = task.id.0, error = %err, "review agent failed");
1094        let stored_error = format_review_error(&ctx.review_agent, &err);
1095        complete_review_run(
1096            store,
1097            run.run_id,
1098            Some(exit_code),
1099            &raw_output,
1100            &stderr,
1101            "",
1102            outcome_for_agent_error(&err),
1103            &stored_error,
1104            token_usage,
1105        )
1106        .await;
1107        return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1108    }
1109
1110    let text_for_review = agent_result
1111        .as_ref()
1112        .map(|r| r.result_text.clone())
1113        .filter(|text| !text.is_empty())
1114        .unwrap_or_else(|| {
1115            tracing::debug!(
1116                task_id = task.id.0,
1117                agent = %ctx.review_agent,
1118                "review agent: per-agent extractor returned no text, using raw output"
1119            );
1120            raw_output.clone()
1121        });
1122
1123    let review_response = match runner::response::parse_review_response(&text_for_review) {
1124        Ok(response) => response,
1125        Err(e) => {
1126            if let Some(response) = runner::response::infer_review_response(&text_for_review) {
1127                tracing::warn!(
1128                    task_id = task.id.0,
1129                    agent = %ctx.review_agent,
1130                    "review response parsed via keyword fallback"
1131                );
1132                response
1133            } else {
1134                // Guard: if the raw NDJSON signals a completed session
1135                // (terminal_reason:completed without is_error:true), do NOT run
1136                // generic pattern-based rate-limit detection over it. Token
1137                // counts in the telemetry blob can contain bare "429" values
1138                // that would otherwise be misclassified as HTTP 429 errors.
1139                let ndjson_completed = ndjson_completed_no_error;
1140
1141                // Missing type:result (kimi/minimax) or terminal_reason:completed: recover
1142                // from assistant text. Never MiniMax parse_response, which synthesizes status=done
1143                if ndjson_completed || !has_result_envelope {
1144                    let assistant_text = runner::agents::collect_assistant_messages_text(
1145                        &ctx.review_agent,
1146                        &raw_output,
1147                    );
1148                    if !assistant_text.is_empty() {
1149                        if let Ok(response) =
1150                            runner::response::parse_review_response(&assistant_text)
1151                        {
1152                            tracing::debug!(
1153                                task_id = task.id.0,
1154                                agent = %ctx.review_agent,
1155                                "found review response JSON in assistant message (terminal_reason:completed)"
1156                            );
1157                            let review_notes_for_comment = response.notes.clone();
1158                            let decision = review_decision_from_response(response);
1159                            return ReviewPhase::Ready(ParsedReview {
1160                                run_id: run.run_id,
1161                                exit_code,
1162                                stderr,
1163                                raw_output,
1164                                text_for_review: assistant_text,
1165                                token_usage,
1166                                review_notes_for_comment,
1167                                decision,
1168                            });
1169                        }
1170                        if let Some(response) =
1171                            runner::response::infer_review_response(&assistant_text)
1172                        {
1173                            tracing::warn!(
1174                                task_id = task.id.0,
1175                                agent = %ctx.review_agent,
1176                                "inferred review response from assistant message (terminal_reason:completed)"
1177                            );
1178                            let review_notes_for_comment = response.notes.clone();
1179                            let decision = review_decision_from_response(response);
1180                            return ReviewPhase::Ready(ParsedReview {
1181                                run_id: run.run_id,
1182                                exit_code,
1183                                stderr,
1184                                raw_output,
1185                                text_for_review: assistant_text,
1186                                token_usage,
1187                                review_notes_for_comment,
1188                                decision,
1189                            });
1190                        }
1191                    }
1192                }
1193
1194                // Scan only bounded head/tail windows, not the full review text.
1195                // The full transcript may contain agent work product (e.g. a
1196                // diff or markdown report quoting an `outcome`/`rate_limit`
1197                // table cell) that incidentally matches these patterns; real
1198                // CLI errors appear at the start or end of the output. See
1199                // `classify_from_text` in runner::agents for the same rationale.
1200                let scan_head = runner::agents::patterns::safe_head(
1201                    &text_for_review,
1202                    runner::agents::patterns::RATE_LIMIT_SCAN_HEAD_BYTES,
1203                );
1204                let scan_tail = runner::agents::patterns::safe_tail(
1205                    &text_for_review,
1206                    runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
1207                );
1208
1209                if let Some(err) = (!ndjson_completed)
1210                    .then(|| {
1211                        runner::agents::patterns::detect_rate_limit(scan_head)
1212                            .or_else(|| runner::agents::patterns::detect_rate_limit(scan_tail))
1213                    })
1214                    .flatten()
1215                {
1216                    let runner::agents::AgentError::RateLimit { message } = &err else {
1217                        unreachable!("detect_rate_limit only returns AgentError::RateLimit")
1218                    };
1219                    tracing::warn!(
1220                        task_id = task.id.0,
1221                        agent = %ctx.review_agent,
1222                        "review agent hit rate limit — adding to cooldown"
1223                    );
1224                    // Route through record_review_agent_failure/outcome_for_agent_error
1225                    // (not the generic record_agent_failure_with_message + hardcoded
1226                    // "rate_limit" string) so billing-cycle exhaustion messages caught by
1227                    // this inline text scan get the same model-scoped persistent cooldown
1228                    // and outcome classification as the AgentError-based failure path
1229                    // above (issue #3551, gap left by #3530).
1230                    let summary = runner::fallback::summarize_rate_limit_error(message);
1231                    let stored_error = format!("{} rate limit: {summary}", ctx.review_agent);
1232                    record_review_agent_failure(
1233                        &task.id.0,
1234                        &ctx.review_agent,
1235                        ctx.review_model.as_deref(),
1236                        &err,
1237                    )
1238                    .await;
1239                    complete_review_run(
1240                        store,
1241                        run.run_id,
1242                        Some(exit_code),
1243                        &raw_output,
1244                        &stderr,
1245                        &text_for_review,
1246                        outcome_for_agent_error(&err),
1247                        &stored_error,
1248                        token_usage,
1249                    )
1250                    .await;
1251                    return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1252                }
1253
1254                if let Some(err) = runner::agents::patterns::detect_auth_error(scan_tail) {
1255                    let runner::agents::AgentError::Auth { message } = &err else {
1256                        unreachable!("detect_auth_error only returns AgentError::Auth")
1257                    };
1258                    tracing::warn!(
1259                        task_id = task.id.0,
1260                        agent = %ctx.review_agent,
1261                        "review agent hit auth error — adding to cooldown"
1262                    );
1263                    let stored_error = format!("auth error: {message}");
1264                    record_review_agent_failure(
1265                        &task.id.0,
1266                        &ctx.review_agent,
1267                        ctx.review_model.as_deref(),
1268                        &err,
1269                    )
1270                    .await;
1271                    complete_review_run(
1272                        store,
1273                        run.run_id,
1274                        Some(exit_code),
1275                        &raw_output,
1276                        &stderr,
1277                        &text_for_review,
1278                        outcome_for_agent_error(&err),
1279                        &stored_error,
1280                        token_usage,
1281                    )
1282                    .await;
1283                    return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1284                }
1285
1286                // The model was cut off for exceeding its output/reasoning
1287                // token budget (opencode step_finish reason=length), not
1288                // because it produced malformed prose. Like a timeout, this is
1289                // a structural inability to finish review under the current
1290                // budget, so use the persistent 4h-7d cooldown rather than the
1291                // generic success-resettable model failure counter.
1292                if agent_result
1293                    .as_ref()
1294                    .map(|r| r.truncated_by_length)
1295                    .unwrap_or(false)
1296                {
1297                    tracing::warn!(
1298                        task_id = task.id.0,
1299                        agent = %ctx.review_agent,
1300                        model = ?ctx.review_model,
1301                        "review agent output truncated: step_finish reason=length \
1302                         (token budget exceeded before completion)"
1303                    );
1304                    if let Some(model) = ctx.review_model.as_deref() {
1305                        crate::engine::cooldown::record_persistent_model_failure(
1306                            &ctx.review_agent,
1307                            model,
1308                        )
1309                        .await;
1310                    }
1311                    let stored_error = "truncated: review output was cut off before completion \
1312                        (output/reasoning token budget exceeded)"
1313                        .to_string();
1314                    complete_review_run(
1315                        store,
1316                        run.run_id,
1317                        Some(exit_code),
1318                        &raw_output,
1319                        &stderr,
1320                        &text_for_review,
1321                        "truncated",
1322                        &stored_error,
1323                        token_usage,
1324                    )
1325                    .await;
1326                    return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1327                }
1328
1329                // No envelope and last content is thinking/tool_use: retry without
1330                // the persistent format-failure cooldown (issue #3734)
1331                if runner::agents::claude::review_session_exited_mid_thought(&raw_output) {
1332                    tracing::warn!(
1333                        task_id = task.id.0,
1334                        agent = %ctx.review_agent,
1335                        model = ?ctx.review_model,
1336                        "review session exited mid-thought with no result envelope \
1337                         — storing as incomplete and retrying without persistent cooldown"
1338                    );
1339                    let stored_error =
1340                        "incomplete: review session exited before producing a decision".to_string();
1341                    complete_review_run(
1342                        store,
1343                        run.run_id,
1344                        Some(exit_code),
1345                        &raw_output,
1346                        &stderr,
1347                        &text_for_review,
1348                        "incomplete",
1349                        &stored_error,
1350                        token_usage,
1351                    )
1352                    .await;
1353                    return ReviewPhase::EarlyReturn(ReviewDecision::Failed(stored_error));
1354                }
1355
1356                tracing::error!(
1357                    task_id = task.id.0,
1358                    error = %e,
1359                    agent = %ctx.review_agent,
1360                    output = %text_for_review.chars().take(300).collect::<String>(),
1361                    raw_output = %raw_output.chars().take(1000).collect::<String>(),
1362                    "failed to parse review response"
1363                );
1364                if let Some(model) = ctx.review_model.as_deref() {
1365                    // Finished output that is not a review decision: persistent backoff
1366                    // (4h base → 7d max) so a later success cannot immediately re-select it
1367                    crate::engine::cooldown::record_persistent_model_failure(
1368                        &ctx.review_agent,
1369                        model,
1370                    )
1371                    .await;
1372                }
1373                complete_review_run(
1374                    store,
1375                    run.run_id,
1376                    Some(exit_code),
1377                    &raw_output,
1378                    &stderr,
1379                    &text_for_review,
1380                    "parse_error",
1381                    &format!("parse error: {e}"),
1382                    token_usage,
1383                )
1384                .await;
1385                return ReviewPhase::EarlyReturn(ReviewDecision::Failed(format!(
1386                    "parse error: {e}"
1387                )));
1388            }
1389        }
1390    };
1391
1392    let review_notes_for_comment = review_response.notes.clone();
1393    let decision = review_decision_from_response(review_response);
1394
1395    tracing::info!(
1396        task_id = task.id.0,
1397        pr_number = ctx.pr_number,
1398        decision = ?decision,
1399        "review agent decision received"
1400    );
1401    store_log_activity(
1402        &Some(Arc::clone(store)),
1403        repo,
1404        &task.id.0,
1405        "review_decision",
1406        None,
1407        None,
1408        Some(&ctx.review_agent),
1409        ctx.review_model.as_deref(),
1410        Some(&serde_json::json!({
1411            "decision": review_decision_name(&decision),
1412            "pr_number": ctx.pr_number,
1413        })),
1414    )
1415    .await;
1416
1417    ReviewPhase::Ready(ParsedReview {
1418        run_id: run.run_id,
1419        exit_code,
1420        stderr,
1421        raw_output,
1422        text_for_review,
1423        token_usage,
1424        review_notes_for_comment,
1425        decision,
1426    })
1427}
1428
1429async fn finalize_review_run(
1430    task: &ExternalTask,
1431    ctx: &ReviewContext,
1432    parsed: &ParsedReview,
1433    store: &Arc<TaskStore>,
1434) {
1435    let outcome = match &parsed.decision {
1436        ReviewDecision::Approve
1437        | ReviewDecision::RequestChanges { .. }
1438        | ReviewDecision::Skipped
1439        | ReviewDecision::Rerouted => "success",
1440        ReviewDecision::Failed(_) | ReviewDecision::Blocked(_) => "failed",
1441    };
1442    let error = match &parsed.decision {
1443        ReviewDecision::Failed(error) | ReviewDecision::Blocked(error) => error.as_str(),
1444        _ => "",
1445    };
1446
1447    if outcome == "success" {
1448        // Mirror the runner's success path so review-only models don't ratchet
1449        // into multi-day cooldowns from isolated failures never offset by resets.
1450        crate::engine::cooldown::record_agent_success(&ctx.review_agent, ctx.review_model_str())
1451            .await;
1452    }
1453
1454    complete_review_run(
1455        store,
1456        parsed.run_id,
1457        Some(parsed.exit_code),
1458        &parsed.raw_output,
1459        &parsed.stderr,
1460        &parsed.text_for_review,
1461        outcome,
1462        error,
1463        parsed.token_usage,
1464    )
1465    .await;
1466
1467    if let Some(store_id) = ctx.store_id {
1468        if parsed.token_usage.input_tokens > 0 || parsed.token_usage.output_tokens > 0 {
1469            let model = if ctx.review_model_str().is_empty() {
1470                "unknown"
1471            } else {
1472                ctx.review_model_str()
1473            };
1474            if let Err(e) = store
1475                .store_tokens(
1476                    store_id,
1477                    parsed.token_usage.input_tokens,
1478                    parsed.token_usage.output_tokens,
1479                    model,
1480                )
1481                .await
1482            {
1483                tracing::warn!(
1484                    task_id = task.id.0,
1485                    ?e,
1486                    "failed to store review agent token usage"
1487                );
1488            }
1489        }
1490    }
1491}
1492
1493async fn restore_review_config_if_needed(
1494    task_id: &str,
1495    worktree_path: &std::path::Path,
1496    default_branch: &str,
1497) {
1498    let _ = Command::new("git")
1499        .args(["checkout", "HEAD", "--", ".orch.yml"])
1500        .current_dir(worktree_path)
1501        .status()
1502        .await;
1503
1504    let merge_base = Command::new("git")
1505        .args(["merge-base", "HEAD", default_branch])
1506        .current_dir(worktree_path)
1507        .output()
1508        .await
1509        .ok()
1510        .filter(|output| output.status.success())
1511        .and_then(|output| String::from_utf8(output.stdout).ok())
1512        .map(|s| s.trim().to_string())
1513        .unwrap_or_default();
1514
1515    if merge_base.is_empty() {
1516        return;
1517    }
1518
1519    let orch_yml_changed = Command::new("git")
1520        .args([
1521            "diff",
1522            "--name-only",
1523            &merge_base,
1524            "HEAD",
1525            "--",
1526            ".orch.yml",
1527        ])
1528        .current_dir(worktree_path)
1529        .output()
1530        .await
1531        .ok()
1532        .filter(|output| output.status.success())
1533        .map(|output| !output.stdout.is_empty())
1534        .unwrap_or(false);
1535
1536    if !orch_yml_changed {
1537        return;
1538    }
1539
1540    tracing::warn!(
1541        task_id,
1542        "review agent modified .orch.yml (forbidden); restoring to merge-base version"
1543    );
1544    let restore_ok = Command::new("git")
1545        .args(["checkout", &merge_base, "--", ".orch.yml"])
1546        .current_dir(worktree_path)
1547        .status()
1548        .await
1549        .map(|status| status.success())
1550        .unwrap_or(false);
1551
1552    if !restore_ok {
1553        return;
1554    }
1555
1556    let staged = Command::new("git")
1557        .args(["diff", "--cached", "--name-only", "--", ".orch.yml"])
1558        .current_dir(worktree_path)
1559        .output()
1560        .await
1561        .ok()
1562        .filter(|output| output.status.success())
1563        .map(|output| !output.stdout.is_empty())
1564        .unwrap_or(false);
1565
1566    if staged {
1567        let _ = Command::new("git")
1568            .args([
1569                "-c",
1570                "commit.gpgsign=false",
1571                "commit",
1572                "-m",
1573                "revert: restore .orch.yml (review agent must not modify project config)",
1574            ])
1575            .current_dir(worktree_path)
1576            .status()
1577            .await;
1578    }
1579}
1580
1581fn is_missing_path_error_message(err: &str) -> bool {
1582    err.to_ascii_lowercase()
1583        .contains("no such file or directory")
1584}
1585
1586async fn push_review_branch(
1587    task: &ExternalTask,
1588    repo: &str,
1589    ctx: &ReviewContext,
1590    store: &Arc<TaskStore>,
1591) -> Result<(), String> {
1592    let mut worktree_path = ctx.worktree_path.clone();
1593    let mut branch_name = ctx.branch_name.clone();
1594    let mut retried_missing_worktree = false;
1595
1596    let push_result = loop {
1597        match runner::git_ops::push_branch(&worktree_path, &branch_name, &ctx.default_branch).await
1598        {
1599            Ok(ok) => break Ok(ok),
1600            Err(e)
1601                if !retried_missing_worktree
1602                    && !worktree_path.exists()
1603                    && is_missing_path_error_message(&e.to_string()) =>
1604            {
1605                retried_missing_worktree = true;
1606                tracing::warn!(
1607                    task_id = task.id.0,
1608                    worktree = %worktree_path.display(),
1609                    branch = %branch_name,
1610                    "review push failed because worktree disappeared; attempting one-time recovery"
1611                );
1612                match recover_missing_review_worktree(task, repo, store).await {
1613                    Ok(recovered) => {
1614                        worktree_path = recovered.work_dir;
1615                        branch_name = recovered.branch;
1616                        continue;
1617                    }
1618                    Err(recover_err) => {
1619                        break Err(anyhow::anyhow!(
1620                            "missing review worktree recovery failed after push error: {recover_err}"
1621                        ));
1622                    }
1623                }
1624            }
1625            Err(e) => break Err(e),
1626        }
1627    };
1628
1629    match push_result {
1630        Ok(_) => {
1631            store_log_activity(
1632                &Some(Arc::clone(store)),
1633                repo,
1634                &task.id.0,
1635                "push",
1636                None,
1637                None,
1638                Some(&ctx.review_agent),
1639                ctx.review_model.as_deref(),
1640                Some(&serde_json::json!({
1641                    "status": "ok",
1642                    "branch": branch_name.clone(),
1643                    "default_branch": ctx.default_branch.clone(),
1644                    "phase": "review",
1645                })),
1646            )
1647            .await;
1648            if ctx.had_prev_push_failure {
1649                if let Err(e) = store_set_result(
1650                    &Some(Arc::clone(store)),
1651                    repo,
1652                    &task.id.0,
1653                    &[
1654                        ("last_error", serde_json::json!("")),
1655                        ("push_failures", serde_json::json!(0)),
1656                    ],
1657                )
1658                .await
1659                {
1660                    tracing::warn!(task_id = task.id.0, err = %e, "failed to reset push_failures — counter may drift");
1661                }
1662            }
1663            Ok(())
1664        }
1665        Err(e) => {
1666            store_log_activity(
1667                &Some(Arc::clone(store)),
1668                repo,
1669                &task.id.0,
1670                "push",
1671                None,
1672                None,
1673                Some(&ctx.review_agent),
1674                ctx.review_model.as_deref(),
1675                Some(&serde_json::json!({
1676                    "status": "error",
1677                    "branch": branch_name.clone(),
1678                    "default_branch": ctx.default_branch.clone(),
1679                    "phase": "review",
1680                    "error": e.to_string(),
1681                })),
1682            )
1683            .await;
1684            tracing::error!(
1685                task_id = task.id.0,
1686                branch = %ctx.branch_name,
1687                error = %e,
1688                "review push failed"
1689            );
1690            store_set(
1691                &Some(Arc::clone(store)),
1692                repo,
1693                &task.id.0,
1694                &[("last_error", serde_json::json!(format!("push failed: {e}")))],
1695            )
1696            .await;
1697            Err(e.to_string())
1698        }
1699    }
1700}
1701
1702fn build_pr_review_comment(decision: &ReviewDecision, review_notes_for_comment: &str) -> String {
1703    match decision {
1704        ReviewDecision::Approve => {
1705            format!(
1706                "## Automated Review \u{2014} Approve\n\n{}",
1707                review_notes_for_comment
1708            )
1709        }
1710        ReviewDecision::RequestChanges { notes, issues } => {
1711            let mut body = format!(
1712                "## Automated Review \u{2014} Changes Requested\n\n{}\n",
1713                notes
1714            );
1715            if !issues.is_empty() {
1716                body.push_str("\n**Issues Found:**\n");
1717                for issue in issues {
1718                    body.push_str(&format!(
1719                        "- `{}` line {}: {} [{}]\n",
1720                        issue.file,
1721                        issue
1722                            .line
1723                            .map(|line| line.to_string())
1724                            .unwrap_or_else(|| "?".to_string()),
1725                        issue.description,
1726                        issue.severity
1727                    ));
1728                }
1729            }
1730            body
1731        }
1732        _ => String::new(),
1733    }
1734}
1735
1736const REVIEW_STATUS_CONTEXT: &str = "orch/review";
1737
1738/// Commit status state and description for a review decision. Other decisions post nothing.
1739fn review_status(decision: &ReviewDecision) -> Option<(&'static str, &'static str)> {
1740    match decision {
1741        ReviewDecision::Approve => Some(("success", "Approved by automated review")),
1742        ReviewDecision::RequestChanges { .. } => {
1743            Some(("failure", "Automated review requested changes"))
1744        }
1745        _ => None,
1746    }
1747}
1748
1749/// Post `orch/review` on the SHA the review started from. A new push moves the PR head, so an
1750/// old decision never covers a new commit.
1751async fn post_review_status(
1752    gh: &GhHttp,
1753    repo: &str,
1754    sha: Option<&str>,
1755    state: &str,
1756    description: &str,
1757) {
1758    let Some(sha) = sha else {
1759        return;
1760    };
1761    if let Err(e) = gh
1762        .post_commit_status(repo, sha, state, REVIEW_STATUS_CONTEXT, description)
1763        .await
1764    {
1765        tracing::warn!(sha, error = %e, "failed to post orch/review commit status");
1766    }
1767}
1768
1769async fn post_review_comment(
1770    task: &ExternalTask,
1771    repo: &str,
1772    ctx: &ReviewContext,
1773    decision: &ReviewDecision,
1774    review_notes_for_comment: &str,
1775) -> anyhow::Result<()> {
1776    let gh = match GhHttp::new() {
1777        Ok(gh) => gh,
1778        Err(e) => {
1779            tracing::warn!(
1780                task_id = task.id.0,
1781                error = %e,
1782                "failed to create GH client for review comment"
1783            );
1784            return Ok(());
1785        }
1786    };
1787    let mut pr_comment = build_pr_review_comment(decision, review_notes_for_comment);
1788    if ctx.same_agent_fallback && !pr_comment.is_empty() {
1789        pr_comment.push_str(
1790            "\n> Note: no other agent was available, so this PR was reviewed by the same agent that wrote the code.\n",
1791        );
1792    }
1793
1794    if let Some((state, description)) = review_status(decision) {
1795        post_review_status(&gh, repo, ctx.head_sha.as_deref(), state, description).await;
1796    }
1797
1798    if !pr_comment.is_empty() {
1799        let footer =
1800            attribution_footer("Reviewed", &ctx.review_agent, Some(ctx.review_model_str()));
1801        let pr_comment_with_footer = format!("{}{}", pr_comment, footer);
1802        if let Err(e) = gh
1803            .add_comment(repo, &ctx.pr_number.to_string(), &pr_comment_with_footer)
1804            .await
1805        {
1806            tracing::warn!(
1807                task_id = task.id.0,
1808                pr_number = ctx.pr_number,
1809                error = %e,
1810                "failed to post automated review comment on PR"
1811            );
1812        }
1813    }
1814
1815    Ok(())
1816}
1817
1818#[allow(clippy::too_many_arguments)]
1819async fn apply_review_decision(
1820    task: &ExternalTask,
1821    backend: &Arc<dyn ExternalBackend>,
1822    repo: &str,
1823    task_manager: &Arc<TaskManager>,
1824    store: &Arc<TaskStore>,
1825    ctx: &ReviewContext,
1826    parsed: &ParsedReview,
1827    router_config: &crate::engine::router::RouterConfig,
1828) -> anyhow::Result<ReviewDecision> {
1829    restore_review_config_if_needed(&task.id.0, &ctx.worktree_path, &ctx.default_branch).await;
1830
1831    if let Err(e) = push_review_branch(task, repo, ctx, store).await {
1832        return Ok(ReviewDecision::Failed(format!("push failed: {e}")));
1833    }
1834
1835    post_review_comment(
1836        task,
1837        repo,
1838        ctx,
1839        &parsed.decision,
1840        &parsed.review_notes_for_comment,
1841    )
1842    .await?;
1843
1844    match &parsed.decision {
1845        ReviewDecision::Approve => {
1846            // Always attempt auto-merge on approval. Tasks are only marked
1847            // Done when the PR is actually merged (detected by review_poll).
1848            // Never mark Done on approval alone — that closes the issue
1849            // before the code lands.
1850            if let Err(e) = auto_merge_pr(
1851                task,
1852                &ctx.branch_name,
1853                backend,
1854                repo,
1855                &ctx.review_agent,
1856                ctx.review_model_str(),
1857                task_manager,
1858                store,
1859            )
1860            .await
1861            {
1862                let error_msg = e.to_string();
1863                if error_msg.contains("not yet computed") {
1864                    tracing::warn!(
1865                        task_id = task.id.0,
1866                        pr_number = ctx.pr_number,
1867                        branch = %ctx.branch_name,
1868                        error = %e,
1869                        "auto-merge deferred — PR mergeability not yet computed"
1870                    );
1871                } else {
1872                    tracing::error!(
1873                        task_id = task.id.0,
1874                        pr_number = ctx.pr_number,
1875                        branch = %ctx.branch_name,
1876                        error = %e,
1877                        "auto-merge failed"
1878                    );
1879                }
1880                return Ok(ReviewDecision::Failed(format!("merge failed: {e}")));
1881            }
1882            Ok(ReviewDecision::Approve)
1883        }
1884        ReviewDecision::RequestChanges { notes, issues } => {
1885            handle_review_changes(
1886                task,
1887                notes,
1888                issues,
1889                backend,
1890                repo,
1891                ctx.pr_number,
1892                &ctx.review_agent,
1893                ctx.review_model_str(),
1894                task_manager,
1895                store,
1896                router_config,
1897            )
1898            .await?;
1899            Ok(parsed.decision.clone())
1900        }
1901        _ => Ok(parsed.decision.clone()),
1902    }
1903}
1904
1905async fn recover_missing_review_worktree(
1906    task: &ExternalTask,
1907    repo: &str,
1908    store: &Arc<TaskStore>,
1909) -> anyhow::Result<worktree::WorktreeSetup> {
1910    let project_dir = crate::config::get_projects_with_paths()?
1911        .into_iter()
1912        .find(|(candidate_repo, _)| candidate_repo == repo)
1913        .map(|(_, project_dir)| project_dir)
1914        .ok_or_else(|| anyhow::anyhow!("no project directory configured for repo {repo}"))?;
1915
1916    tracing::warn!(
1917        task_id = task.id.0,
1918        repo,
1919        project_dir = %project_dir.display(),
1920        "review worktree missing, recreating via normal setup"
1921    );
1922
1923    let store_ref = Some(Arc::clone(store));
1924    let wt = worktree::setup_worktree(&task.id.0, &task.title, &project_dir, &store_ref, repo)
1925        .await
1926        .context("recreating missing review worktree")?;
1927
1928    if !wt.work_dir.exists() {
1929        anyhow::bail!(
1930            "recreated review worktree at {} for task {} is still missing",
1931            wt.work_dir.display(),
1932            task.id.0
1933        );
1934    }
1935
1936    Ok(wt)
1937}
1938
1939/// Count commits on the worktree branch that are ahead of `origin/<default_branch>`.
1940///
1941/// Returns `Err` when the git command cannot determine the count (e.g., missing
1942/// remote ref, non-zero exit, unparseable output). Callers must NOT treat this
1943/// as zero — a failure means the ancestry relationship is unknown.
1944async fn count_ahead_commits(
1945    worktree_path: &std::path::Path,
1946    default_branch: &str,
1947) -> Result<u64, String> {
1948    let worktree_str = worktree_path.to_str().ok_or_else(|| {
1949        format!(
1950            "worktree path contains non-UTF-8 characters: {:?}",
1951            worktree_path
1952        )
1953    })?;
1954
1955    let output = tokio::process::Command::new("git")
1956        .args([
1957            "-C",
1958            worktree_str,
1959            "rev-list",
1960            "--count",
1961            &format!("origin/{default_branch}..HEAD"),
1962        ])
1963        .output()
1964        .await
1965        .map_err(|e| format!("git rev-list command failed to start: {e}"))?;
1966
1967    if !output.status.success() {
1968        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
1969        return Err(format!(
1970            "git rev-list exited with {:?}: {}",
1971            output.status.code(),
1972            if stderr.is_empty() {
1973                String::from_utf8_lossy(&output.stdout).trim().to_string()
1974            } else {
1975                stderr
1976            }
1977        ));
1978    }
1979
1980    String::from_utf8_lossy(&output.stdout)
1981        .trim()
1982        .parse::<u64>()
1983        .map_err(|e| format!("could not parse git rev-list output as u64: {e}"))
1984}
1985
1986/// Ensure an open PR exists for the task branch before running the review agent.
1987///
1988/// Checks the store first (avoids GitHub list-API cache race), then the API.
1989/// If no PR exists but the branch has commits, attempts to create one and
1990/// continues with the same review pass.
1991/// All error and no-op paths update task status and return [`EnsurePrResult::EarlyReturn`].
1992#[allow(clippy::too_many_arguments)]
1993async fn ensure_pr_exists(
1994    task: &ExternalTask,
1995    branch_name: &str,
1996    worktree_path: &std::path::Path,
1997    repo: &str,
1998    agent_summary: &str,
1999    store: &Arc<TaskStore>,
2000    task_manager: &Arc<TaskManager>,
2001    stored_pr_number: Option<u64>,
2002) -> anyhow::Result<EnsurePrResult> {
2003    let gh_check = GhHttp::new()?;
2004
2005    if let Some(n) = stored_pr_number {
2006        tracing::info!(
2007            task_id = task.id.0,
2008            pr_number = n,
2009            branch = %branch_name,
2010            "open PR found in store, proceeding with review"
2011        );
2012        return Ok(EnsurePrResult::Ready(n));
2013    }
2014
2015    match gh_check.get_pr_number(repo, branch_name).await {
2016        Ok(Some(n)) => {
2017            tracing::info!(
2018                task_id = task.id.0,
2019                pr_number = n,
2020                branch = %branch_name,
2021                "open PR found, proceeding with review"
2022            );
2023            Ok(EnsurePrResult::Ready(n))
2024        }
2025        Ok(None) => {
2026            // No open PR — check if branch has commits ahead of default branch.
2027            let default_branch = worktree::detect_default_branch(worktree_path).await;
2028            let has_commits = match count_ahead_commits(worktree_path, &default_branch).await {
2029                Ok(count) => count > 0,
2030                Err(e) => {
2031                    tracing::error!(
2032                        task_id = task.id.0,
2033                        branch = %branch_name,
2034                        default_branch = %default_branch,
2035                        err = %e,
2036                        "could not determine if branch has commits ahead of default"
2037                    );
2038                    return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2039                        format!("git ancestry check failed: {e}"),
2040                    )));
2041                }
2042            };
2043
2044            if has_commits {
2045                // Branch has unpushed or un-PR'd work — try to create a PR
2046                tracing::warn!(
2047                    task_id = task.id.0,
2048                    branch = %branch_name,
2049                    "no open PR but branch has commits — attempting to create PR"
2050                );
2051                let worktree_str = worktree_path.to_str().ok_or_else(|| {
2052                    anyhow::anyhow!(
2053                        "worktree path contains non-UTF-8 characters: {:?}",
2054                        worktree_path
2055                    )
2056                })?;
2057                // Push first in case agent forgot (use token auth like git_ops::push_branch)
2058                let auth_env = runner::git_ops::build_git_auth_env();
2059
2060                // Run a credentialed push and log any errors so failures are
2061                // visible in logs instead of being silently discarded.
2062                // Wrap with a timeout to prevent indefinitely blocking a Tokio worker on a
2063                // hung network connection (corporate proxy, partial GitHub outage, TCP hang).
2064                let push_timeout = std::time::Duration::from_secs(120);
2065                match tokio::time::timeout(
2066                    push_timeout,
2067                    tokio::process::Command::new("git")
2068                        .args(["-C", worktree_str, "push", "-u", "origin", branch_name])
2069                        .envs(auth_env.iter().map(|(k, v)| (k.as_str(), v.as_str())))
2070                        .output(),
2071                )
2072                .await
2073                {
2074                    Err(_elapsed) => {
2075                        tracing::warn!(
2076                            task_id = task.id.0,
2077                            branch = %branch_name,
2078                            timeout_secs = push_timeout.as_secs(),
2079                            "review gate fallback push timed out"
2080                        );
2081                    }
2082                    Ok(Ok(o)) => {
2083                        if !o.status.success() {
2084                            tracing::warn!(
2085                                task_id = task.id.0,
2086                                branch = %branch_name,
2087                                exit_code = ?o.status.code(),
2088                                stdout = %String::from_utf8_lossy(&o.stdout),
2089                                stderr = %String::from_utf8_lossy(&o.stderr),
2090                                "review gate fallback push failed"
2091                            );
2092                        } else {
2093                            tracing::info!(task_id = task.id.0, branch = %branch_name, "review gate fallback push succeeded");
2094                        }
2095                    }
2096                    Ok(Err(e)) => {
2097                        tracing::error!(
2098                            task_id = task.id.0,
2099                            branch = %branch_name,
2100                            error = %e,
2101                            "review gate fallback push command failed to run"
2102                        );
2103                    }
2104                }
2105
2106                let task_ref = runner::git_ops::format_task_ref(&task.id.0);
2107                let pr_body = format!(
2108                    "Resolves {task_ref}\n\nAuto-created by orch review gate (agent forgot to open PR)"
2109                );
2110                let gh = GhHttp::new()?;
2111                match gh
2112                    .create_pr(repo, &task.title, &pr_body, branch_name, &default_branch)
2113                    .await
2114                {
2115                    Ok(url) => match parse_pr_number_from_url(&url) {
2116                        Ok(pr_num) => {
2117                            store_set_result(
2118                                &Some(Arc::clone(store)),
2119                                repo,
2120                                &task.id.0,
2121                                &[("pr_number", serde_json::json!(pr_num as i64))],
2122                            )
2123                            .await?;
2124                            tracing::info!(
2125                                task_id = task.id.0,
2126                                branch = %branch_name,
2127                                pr_url = %url,
2128                                "created missing PR via GhHttp — continuing review"
2129                            );
2130                            Ok(EnsurePrResult::Ready(pr_num))
2131                        }
2132                        Err(e) => {
2133                            tracing::warn!(
2134                                task_id = task.id.0,
2135                                branch = %branch_name,
2136                                pr_url = %url,
2137                                error = %e,
2138                                "created missing PR via GhHttp, but failed to parse PR number"
2139                            );
2140                            Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2141                                format!("created missing PR, but could not parse PR number: {e}"),
2142                            )))
2143                        }
2144                    },
2145                    Err(e) => {
2146                        let e_str = format!("{e}");
2147                        if e_str.contains("already exists") {
2148                            // GitHub list-API cache may be stale immediately after a 422.
2149                            // Use two attempts with exponential backoff (2s → 4s) to avoid
2150                            // racing into a duplicate `gh pr create` call.
2151                            let backoff_delays = [2u64, 4u64];
2152                            let mut found_pr: Option<u64> = None;
2153                            for delay_secs in backoff_delays {
2154                                tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs))
2155                                    .await;
2156                                if let Ok(Some(n)) = gh_check.get_pr_number(repo, branch_name).await
2157                                {
2158                                    found_pr = Some(n);
2159                                    break;
2160                                }
2161                            }
2162                            if let Some(n) = found_pr {
2163                                tracing::info!(
2164                                    task_id = task.id.0,
2165                                    pr_number = n,
2166                                    branch = %branch_name,
2167                                    "found existing PR after create_pr 422 — retrying review"
2168                                );
2169                                store_set_result(
2170                                    &Some(Arc::clone(store)),
2171                                    repo,
2172                                    &task.id.0,
2173                                    &[("pr_number", serde_json::json!(n as i64))],
2174                                )
2175                                .await?;
2176                                return Ok(EnsurePrResult::Ready(n));
2177                            }
2178                        }
2179                        tracing::warn!(
2180                            task_id = task.id.0,
2181                            branch = %branch_name,
2182                            error = %e,
2183                            "create_pr failed via GhHttp, falling back to CLI"
2184                        );
2185                        // Fall back to CLI. Wrap the `gh pr create` call in a timeout so
2186                        // a hung CLI/network does not block the review pipeline.
2187                        let pr_timeout = std::time::Duration::from_secs(120);
2188                        let mut gh_cmd = tokio::process::Command::new("gh");
2189                        gh_cmd.args([
2190                            "pr",
2191                            "create",
2192                            "--repo",
2193                            repo,
2194                            "--head",
2195                            branch_name,
2196                            "--title",
2197                            &task.title,
2198                            "--body",
2199                            &pr_body,
2200                        ]);
2201                        gh_cmd.current_dir(worktree_path);
2202                        gh_cmd.kill_on_drop(true);
2203
2204                        let pr_result = tokio::time::timeout(pr_timeout, gh_cmd.output()).await;
2205                        match pr_result {
2206                            Ok(Ok(o)) if o.status.success() => {
2207                                let stdout = String::from_utf8_lossy(&o.stdout);
2208                                match parse_pr_number_from_url(stdout.trim()) {
2209                                    Ok(pr_num) => {
2210                                        store_set_result(
2211                                            &Some(Arc::clone(store)),
2212                                            repo,
2213                                            &task.id.0,
2214                                            &[("pr_number", serde_json::json!(pr_num as i64))],
2215                                        )
2216                                        .await?;
2217                                        tracing::info!(
2218                                            task_id = task.id.0,
2219                                            branch = %branch_name,
2220                                            "created missing PR via CLI — continuing review"
2221                                        );
2222                                        Ok(EnsurePrResult::Ready(pr_num))
2223                                    }
2224                                    Err(e) => {
2225                                        tracing::warn!(
2226                                            task_id = task.id.0,
2227                                            branch = %branch_name,
2228                                            stdout = %stdout,
2229                                            error = %e,
2230                                            "created missing PR via CLI, but failed to parse PR number"
2231                                        );
2232                                        Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2233                                            format!("created missing PR via CLI, but could not parse PR number: {e}")
2234                                        )))
2235                                    }
2236                                }
2237                            }
2238                            Ok(Ok(o)) => {
2239                                let stderr = String::from_utf8_lossy(&o.stderr);
2240                                if stderr.contains("already exists") {
2241                                    if let Some(pr_url) =
2242                                        stderr.lines().find(|l| l.trim().starts_with("https://"))
2243                                    {
2244                                        match parse_pr_number_from_url(pr_url.trim()) {
2245                                            Ok(pr_num) => {
2246                                                if let Err(e) = store_set_result(
2247                                                    &Some(Arc::clone(store)),
2248                                                    repo,
2249                                                    &task.id.0,
2250                                                    &[(
2251                                                        "pr_number",
2252                                                        serde_json::json!(pr_num as i64),
2253                                                    )],
2254                                                )
2255                                                .await
2256                                                {
2257                                                    tracing::warn!(
2258                                                        task_id = task.id.0,
2259                                                        pr_num,
2260                                                        err = %e,
2261                                                        "failed to persist pr_number from already-exists recovery — downstream may retry PR creation"
2262                                                    );
2263                                                }
2264                                                tracing::info!(
2265                                                    task_id = task.id.0,
2266                                                    branch = %branch_name,
2267                                                    pr_url = %pr_url,
2268                                                    "PR already exists (from CLI stderr) — retrying review"
2269                                                );
2270                                                return Ok(EnsurePrResult::Ready(pr_num));
2271                                            }
2272                                            Err(e) => {
2273                                                tracing::warn!(
2274                                                    task_id = task.id.0,
2275                                                    branch = %branch_name,
2276                                                    pr_url = %pr_url,
2277                                                    error = %e,
2278                                                    "found existing PR URL in CLI stderr, but failed to parse PR number"
2279                                                );
2280                                                // Continue to the error handling below
2281                                            }
2282                                        }
2283                                    }
2284                                }
2285                                tracing::error!(
2286                                    task_id = task.id.0,
2287                                    branch = %branch_name,
2288                                    stderr = %stderr,
2289                                    "failed to create missing PR — work may be stuck"
2290                                );
2291
2292                                // If this was a transient GitHub 5xx or transport error,
2293                                // do NOT increment the persistent pr_create_failures counter
2294                                // (which would eventually block the task). Let the
2295                                // engine retry later. Detect via git_ops::is_transient_github_api_error
2296                                // so generic agent/model transport errors (e.g. broken pipe)
2297                                // are NOT mis-classified as GitHub API transients.
2298                                if crate::engine::runner::git_ops::is_transient_github_api_error(
2299                                    &stderr,
2300                                ) {
2301                                    tracing::warn!(
2302                                        task_id = task.id.0,
2303                                        branch = %branch_name,
2304                                        "transient GitHub error creating PR; will retry later without incrementing persistent failure counter"
2305                                    );
2306                                    return Ok(EnsurePrResult::EarlyReturn(
2307                                        ReviewDecision::Failed(format!(
2308                                            "transient github error creating PR: {stderr}"
2309                                        )),
2310                                    ));
2311                                }
2312
2313                                match store_increment(
2314                                    &Some(Arc::clone(store)),
2315                                    repo,
2316                                    &task.id.0,
2317                                    "pr_create_failures",
2318                                )
2319                                .await
2320                                {
2321                                    Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2322                                        return Ok(EnsurePrResult::EarlyReturn(
2323                                            ReviewDecision::Blocked(format!(
2324                                                "no PR, create failed {failures} times: {stderr}"
2325                                            )),
2326                                        ));
2327                                    }
2328                                    Ok(_) => {}
2329                                    Err(e) => {
2330                                        tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2331                                    }
2332                                }
2333                                Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2334                                    format!("no PR, create failed: {stderr}"),
2335                                )))
2336                            }
2337                            Ok(Err(e)) => {
2338                                tracing::error!(
2339                                    task_id = task.id.0,
2340                                    error = %e,
2341                                    "failed to run gh pr create"
2342                                );
2343                                let e_str = format!("{e}");
2344                                if crate::engine::runner::git_ops::is_transient_github_api_error(
2345                                    &e_str,
2346                                ) {
2347                                    tracing::warn!(
2348                                        task_id = task.id.0,
2349                                        branch = %branch_name,
2350                                        "transient GitHub error from gh CLI fallback; will retry later without incrementing persistent failure counter"
2351                                    );
2352                                    return Ok(EnsurePrResult::EarlyReturn(
2353                                        ReviewDecision::Failed(format!("transient gh error: {e}")),
2354                                    ));
2355                                }
2356
2357                                match store_increment(
2358                                    &Some(Arc::clone(store)),
2359                                    repo,
2360                                    &task.id.0,
2361                                    "pr_create_failures",
2362                                )
2363                                .await
2364                                {
2365                                    Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2366                                        return Ok(EnsurePrResult::EarlyReturn(
2367                                            ReviewDecision::Blocked(format!(
2368                                                "no PR, gh error {failures} times: {e}"
2369                                            )),
2370                                        ));
2371                                    }
2372                                    Ok(_) => {}
2373                                    Err(e) => {
2374                                        tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2375                                    }
2376                                }
2377                                Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2378                                    format!("no PR, gh error: {e}"),
2379                                )))
2380                            }
2381                            Err(_elapsed) => {
2382                                // Timeout from tokio::time::timeout
2383                                tracing::error!(
2384                                    task_id = task.id.0,
2385                                    timeout_secs = pr_timeout.as_secs(),
2386                                    "gh pr create CLI fallback timed out"
2387                                );
2388                                let e_str = format!(
2389                                    "gh pr create timed out after {}s",
2390                                    pr_timeout.as_secs()
2391                                );
2392                                if crate::engine::runner::git_ops::is_transient_github_api_error(
2393                                    &e_str,
2394                                ) {
2395                                    tracing::warn!(
2396                                        task_id = task.id.0,
2397                                        branch = %branch_name,
2398                                        "transient GitHub timeout from gh CLI fallback; will retry later without incrementing persistent failure counter"
2399                                    );
2400                                    return Ok(EnsurePrResult::EarlyReturn(
2401                                        ReviewDecision::Failed(format!(
2402                                            "transient gh error: {e_str}"
2403                                        )),
2404                                    ));
2405                                }
2406
2407                                match store_increment(
2408                                    &Some(Arc::clone(store)),
2409                                    repo,
2410                                    &task.id.0,
2411                                    "pr_create_failures",
2412                                )
2413                                .await
2414                                {
2415                                    Ok(failures) if failures >= MAX_PR_CREATE_FAILURES => {
2416                                        return Ok(EnsurePrResult::EarlyReturn(
2417                                            ReviewDecision::Blocked(format!(
2418                                                "no PR, gh timeout {failures} times: {e_str}"
2419                                            )),
2420                                        ));
2421                                    }
2422                                    Ok(_) => {}
2423                                    Err(e) => {
2424                                        tracing::warn!(task_id = task.id.0, err = %e, "failed to increment pr_create_failures — skipping blocking decision this tick");
2425                                    }
2426                                }
2427                                Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2428                                    format!("no PR, gh timeout: {e_str}"),
2429                                )))
2430                            }
2431                        }
2432                    }
2433                }
2434            } else {
2435                // No PR and no commits — agent either failed or completed a read-only task.
2436                let merged = match gh_check.is_pr_merged(repo, branch_name).await {
2437                    Ok(v) => v,
2438                    Err(e) => {
2439                        tracing::warn!(task_id = task.id.0, branch = %branch_name, err = %e, "merge check failed, skipping task this tick");
2440                        return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2441                            format!("merge check failed: {e}"),
2442                        )));
2443                    }
2444                };
2445                if merged {
2446                    tracing::info!(
2447                        task_id = task.id.0,
2448                        branch = %branch_name,
2449                        "PR already merged, marking done"
2450                    );
2451                    if let Err(e) = task_manager
2452                        .update_task_status(&task.id, Status::Done)
2453                        .await
2454                    {
2455                        tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed — task may be stuck in InReview");
2456                    }
2457                    return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2458                }
2459
2460                let last_error = opt_store_get_task(&Some(Arc::clone(store)), repo, &task.id.0)
2461                    .await
2462                    .map(|t| t.last_error)
2463                    .unwrap_or_default();
2464                let reason = if !agent_summary.is_empty() {
2465                    agent_summary.to_string()
2466                } else {
2467                    last_error.clone()
2468                };
2469
2470                if last_error.contains("exceeded max attempts") {
2471                    tracing::warn!(
2472                        task_id = task.id.0,
2473                        branch = %branch_name,
2474                        "no PR and no commits after max attempts — marking blocked to stop loop"
2475                    );
2476                    // Write block_reason atomically with the status transition to prevent
2477                    // auto_unblock_blocked_tasks (block_reason.is_none() gate) from
2478                    // re-dispatching this task.
2479                    let fields = [
2480                        (
2481                            "block_reason",
2482                            serde_json::json!("max attempts exceeded — no PR or commits produced"),
2483                        ),
2484                        ("last_error", serde_json::json!(last_error)),
2485                    ];
2486                    if let Err(e) = task_manager
2487                        .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2488                        .await
2489                    {
2490                        tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2491                        return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2492                            format!("failed to write block_reason: {e}"),
2493                        )));
2494                    }
2495                    return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2496                }
2497
2498                if last_error.contains("422") && last_error.contains("head") {
2499                    tracing::info!(
2500                        task_id = task.id.0,
2501                        branch = %branch_name,
2502                        "no PR — 422/head-invalid means work already merged, marking done"
2503                    );
2504                    if let Err(e) = task_manager
2505                        .update_task_status(&task.id, Status::Done)
2506                        .await
2507                    {
2508                        tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed");
2509                    }
2510                    return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2511                }
2512
2513                if last_error.contains("no fallback agents")
2514                    || last_error.contains("all agents exhausted")
2515                {
2516                    tracing::warn!(
2517                        task_id = task.id.0,
2518                        branch = %branch_name,
2519                        last_error = %last_error,
2520                        "no PR and no commits after failover exhaustion — marking blocked to stop loop"
2521                    );
2522                    // Write block_reason atomically with the status transition to prevent
2523                    // auto_unblock_blocked_tasks (block_reason.is_none() gate) from
2524                    // re-dispatching this task.
2525                    let fields = [
2526                        (
2527                            "block_reason",
2528                            serde_json::json!("all agents exhausted — no PR or commits produced"),
2529                        ),
2530                        ("last_error", serde_json::json!(last_error)),
2531                    ];
2532                    if let Err(e) = task_manager
2533                        .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2534                        .await
2535                    {
2536                        tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2537                        return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2538                            format!("failed to write block_reason: {e}"),
2539                        )));
2540                    }
2541                    return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped));
2542                }
2543
2544                // No PR and no commits — agent either failed or completed a read-only task.
2545                // Use a dedicated circuit-breaker counter persisted in the store so
2546                // repeated reroutes across separate runs are counted. This prevents
2547                // internal tasks from looping new→in_progress→needs_review→new indefinitely.
2548                // Prefer a dedicated config key `workflow.max_reroute_attempts` (fallback to
2549                // `workflow.max_attempts` for backwards compatibility).
2550                let max_reroutes: u32 = config::get("workflow.max_reroute_attempts")
2551                    .ok()
2552                    .and_then(|s| s.parse().ok())
2553                    .or_else(|| {
2554                        config::get("workflow.max_attempts")
2555                            .ok()
2556                            .and_then(|s| s.parse().ok())
2557                    })
2558                    .unwrap_or(3);
2559
2560                tracing::warn!(
2561                    task_id = task.id.0,
2562                    branch = %branch_name,
2563                    reason = %reason,
2564                    "no PR and no commits — re-routing for retry"
2565                );
2566
2567                // If the last_error indicates a transient GitHub 5xx/transport error,
2568                // do NOT increment the persistent reroute counter which would
2569                // eventually block the task. Instead, retry later.
2570                if should_skip_no_code_reroute_increment(&last_error) {
2571                    tracing::warn!(
2572                        task_id = task.id.0,
2573                        last_error = %last_error,
2574                        "transient GitHub error recorded; skipping persistent no_pr_reroutes increment"
2575                    );
2576                    return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2577                        format!("transient github error: {last_error}"),
2578                    )));
2579                }
2580
2581                // Atomically increment the persistent reroute counter and decide.
2582                let reroutes = match store_increment(
2583                    &Some(Arc::clone(store)),
2584                    repo,
2585                    &task.id.0,
2586                    "no_code_reroutes",
2587                )
2588                .await
2589                {
2590                    Ok(v) => v,
2591                    Err(e) => {
2592                        tracing::warn!(task_id = task.id.0, err = %e, "failed to increment no_code_reroutes — skipping reroute/block decision this tick");
2593                        // Skip action this tick
2594                        return Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2595                            format!("transient store error: {e}"),
2596                        )));
2597                    }
2598                };
2599
2600                if reroutes as u32 >= max_reroutes {
2601                    tracing::error!(
2602                        task_id = task.id.0,
2603                        reroutes,
2604                        max_reroutes,
2605                        "reached max reroute attempts for no-pr-result — blocking for human review"
2606                    );
2607                    let msg = format!(
2608                        "no PR or code changes after {}/{} reroute attempts",
2609                        reroutes, max_reroutes
2610                    );
2611                    // Write block_reason atomically with the status transition to prevent
2612                    // auto_unblock_blocked_tasks (block_reason.is_none() gate) from
2613                    // re-dispatching this task.
2614                    let fields = [
2615                        (
2616                            "block_reason",
2617                            serde_json::json!(format!(
2618                                "max reroute attempts ({}) reached — no PR or code changes produced",
2619                                max_reroutes
2620                            )),
2621                        ),
2622                        ("last_error", serde_json::json!(msg)),
2623                        ("agent", serde_json::json!(null)),
2624                        ("model", serde_json::json!(null)),
2625                    ];
2626                    if let Err(e) = task_manager
2627                        .update_task_status_and_result(&task.id, Status::Blocked, &fields)
2628                        .await
2629                    {
2630                        tracing::error!(task_id = task.id.0, err = %e, "update_task_status_and_result(Blocked) failed — skipping block to avoid silent auto-unblock loop");
2631                    }
2632                    Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Blocked(msg)))
2633                } else if task.id.0.starts_with("internal:") {
2634                    // Internal tasks with no PR or code changes are read-only/analysis
2635                    // tasks (e.g. morning briefing, evening retrospective) that
2636                    // legitimately produce no git-visible changes. Mark them done
2637                    // instead of re-routing, which would create a loop: needs_review
2638                    // → review agent → no PR/no commits → reroute to New → agent
2639                    // runs → needs_review → ...
2640                    tracing::info!(
2641                        task_id = task.id.0,
2642                        "no PR and no commits for internal task — marking done"
2643                    );
2644                    if let Err(e) = task_manager
2645                        .update_task_status(&task.id, Status::Done)
2646                        .await
2647                    {
2648                        tracing::error!(task_id = task.id.0, err = %e, "update_task_status(Done) failed — task may be stuck in InReview");
2649                    }
2650                    Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Skipped))
2651                } else {
2652                    // Clear agent/model so router picks a different one and note the
2653                    // fact that this attempt produced no PR or code changes.
2654                    // Use store_set_result so we only transition status when the
2655                    // store write succeeds — partial failure would orphan the task.
2656                    if let Err(e) = store_set_result(
2657                        &Some(Arc::clone(store)),
2658                        repo,
2659                        &task.id.0,
2660                        &[
2661                            ("agent", serde_json::json!(null)),
2662                            ("model", serde_json::json!(null)),
2663                            (
2664                                "last_error",
2665                                serde_json::json!("no PR or code changes produced"),
2666                            ),
2667                        ],
2668                    )
2669                    .await
2670                    {
2671                        tracing::error!(task_id = task.id.0, err = %e, "failed to clear agent/model for reroute — skipping status transition");
2672                        // Don't update status — task stays in InReview and will be retried next tick
2673                    } else if let Err(e) =
2674                        task_manager.update_task_status(&task.id, Status::New).await
2675                    {
2676                        tracing::error!(task_id = task.id.0, err = %e, "update_task_status(New) failed — task may be stuck in InReview");
2677                    }
2678                    Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Rerouted))
2679                }
2680            }
2681        }
2682        Err(e) => {
2683            tracing::warn!(
2684                task_id = task.id.0,
2685                branch = %branch_name,
2686                error = %e,
2687                "failed to check PR status"
2688            );
2689            Ok(EnsurePrResult::EarlyReturn(ReviewDecision::Failed(
2690                format!("PR check failed: {e}"),
2691            )))
2692        }
2693    }
2694}
2695
2696/// Run the review agent on a completed task and handle the outcome.
2697///
2698/// Called after a task completes with status:done and a PR is created.
2699/// The review agent checks the changes and either approves (triggers auto-merge)
2700/// or requests changes (re-dispatches the original agent).
2701pub(crate) async fn review_and_merge(
2702    task: &ExternalTask,
2703    backend: &Arc<dyn ExternalBackend>,
2704    tmux: &Arc<TmuxManager>,
2705    repo: &str,
2706    router: &Arc<RwLock<Router>>,
2707    task_manager: &Arc<TaskManager>,
2708    store: &Arc<TaskStore>,
2709) -> anyhow::Result<ReviewDecision> {
2710    set_review_session_expected(store, repo, &task.id.0, true).await;
2711
2712    let ctx = match build_review_context(task, repo, router, task_manager, store).await? {
2713        ReviewPhase::Ready(ctx) => ctx,
2714        ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2715    };
2716
2717    let run = match invoke_review_agent(task, tmux, &ctx, repo, store).await {
2718        ReviewPhase::Ready(run) => run,
2719        ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2720    };
2721
2722    let parsed = match parse_review_output(task, repo, &ctx, run, store).await {
2723        ReviewPhase::Ready(parsed) => parsed,
2724        ReviewPhase::EarlyReturn(decision) => return Ok(decision),
2725    };
2726
2727    finalize_review_run(task, &ctx, &parsed, store).await;
2728
2729    let router_config = router.read().await.config.clone();
2730    apply_review_decision(
2731        task,
2732        backend,
2733        repo,
2734        task_manager,
2735        store,
2736        &ctx,
2737        &parsed,
2738        &router_config,
2739    )
2740    .await
2741}
2742
2743#[cfg(test)]
2744mod tests {
2745    use super::*;
2746
2747    use crate::backends::ExternalTask;
2748    use crate::engine::router::RouterConfig;
2749    use crate::github::types::{GitHubReview, GitHubReviewComment, GitHubUser, PullRequestReview};
2750    use crate::store::{TaskStatus, TaskStore};
2751    use tempfile::TempDir;
2752
2753    // ── outcome_for_agent_error ─────────────────────────────────────────────
2754
2755    #[test]
2756    fn outcome_for_agent_error_kimi_billing_cycle_is_not_rate_limit() {
2757        // Regression test for issue #3529: a review run hitting Kimi's billing-cycle
2758        // exhaustion (task 156734 in the bug report) must not be stored as a plain
2759        // "rate_limit" outcome.
2760        let err = runner::agents::AgentError::RateLimit {
2761            message: "Failed to authenticate. API Error: 403 You've reached your usage limit \
2762                      for this billing cycle. Your quota will be refreshed soon."
2763                .to_string(),
2764        };
2765        assert_eq!(outcome_for_agent_error(&err), "billing_cycle_exhausted");
2766    }
2767
2768    #[test]
2769    fn outcome_for_agent_error_generic_rate_limit_unaffected() {
2770        let err = runner::agents::AgentError::RateLimit {
2771            message: "429 Too Many Requests".to_string(),
2772        };
2773        assert_eq!(outcome_for_agent_error(&err), "rate_limit");
2774    }
2775
2776    #[test]
2777    fn outcome_for_agent_error_generic_auth_unaffected() {
2778        let err = runner::agents::AgentError::Auth {
2779            message: "401 Unauthorized".to_string(),
2780        };
2781        assert_eq!(outcome_for_agent_error(&err), "auth_error");
2782    }
2783
2784    // ── parse_pr_number_from_url ────────────────────────────────────────────
2785
2786    #[test]
2787    fn parse_pr_number_standard_url() {
2788        assert_eq!(
2789            parse_pr_number_from_url("https://github.com/owner/repo/pull/42"),
2790            Ok(42)
2791        );
2792    }
2793
2794    #[test]
2795    fn parse_pr_number_trailing_slash() {
2796        assert_eq!(
2797            parse_pr_number_from_url("https://github.com/owner/repo/pull/42/"),
2798            Ok(42)
2799        );
2800    }
2801
2802    #[test]
2803    fn parse_pr_number_with_query_string() {
2804        assert_eq!(
2805            parse_pr_number_from_url("https://github.com/owner/repo/pull/42?tab=files"),
2806            Ok(42)
2807        );
2808    }
2809
2810    #[test]
2811    fn parse_pr_number_with_hash_fragment() {
2812        assert_eq!(
2813            parse_pr_number_from_url("https://github.com/owner/repo/pull/42#issuecomment-123"),
2814            Ok(42)
2815        );
2816    }
2817
2818    #[test]
2819    fn parse_pr_number_with_query_and_hash() {
2820        assert_eq!(
2821            parse_pr_number_from_url("https://github.com/owner/repo/pull/42?quick_pull=1#top"),
2822            Ok(42)
2823        );
2824    }
2825
2826    #[test]
2827    fn parse_pr_number_with_leading_whitespace() {
2828        assert_eq!(
2829            parse_pr_number_from_url("  https://github.com/owner/repo/pull/42\n"),
2830            Ok(42)
2831        );
2832    }
2833
2834    #[test]
2835    fn parse_pr_number_wrong_domain() {
2836        let url = "https://gitlab.com/owner/repo/merge_requests/42";
2837        let result = parse_pr_number_from_url(url);
2838        assert!(result.is_err());
2839        assert!(result
2840            .unwrap_err()
2841            .contains("does not start with 'https://github.com/'"));
2842    }
2843
2844    #[test]
2845    fn parse_pr_number_missing_pull_segment() {
2846        // Issue URL — not a PR URL
2847        let url = "https://github.com/owner/repo/issues/42";
2848        let result = parse_pr_number_from_url(url);
2849        assert!(result.is_err());
2850        assert!(result.unwrap_err().contains("missing '/pull/' segment"));
2851    }
2852
2853    #[test]
2854    fn parse_pr_number_zero_is_rejected() {
2855        let url = "https://github.com/owner/repo/pull/0";
2856        let result = parse_pr_number_from_url(url);
2857        assert!(result.is_err());
2858        assert!(result.unwrap_err().contains("cannot be zero"));
2859    }
2860
2861    #[test]
2862    fn parse_pr_number_non_numeric_segment() {
2863        let url = "https://github.com/owner/repo/pull/abc";
2864        let result = parse_pr_number_from_url(url);
2865        assert!(result.is_err());
2866        assert!(result.unwrap_err().contains("not a valid positive integer"));
2867    }
2868
2869    #[test]
2870    fn parse_pr_number_empty_string() {
2871        let result = parse_pr_number_from_url("");
2872        assert!(result.is_err());
2873        assert!(result.unwrap_err().contains("empty"));
2874    }
2875
2876    #[test]
2877    fn parse_pr_number_http_not_https() {
2878        // HTTP URLs are not accepted — only HTTPS
2879        let url = "http://github.com/owner/repo/pull/42";
2880        let result = parse_pr_number_from_url(url);
2881        assert!(result.is_err());
2882        assert!(result
2883            .unwrap_err()
2884            .contains("does not start with 'https://github.com/'"));
2885    }
2886
2887    // ── format_review_error ─────────────────────────────────────────────────
2888    //
2889    // Regression coverage for issue #3149: review runs were persisting raw
2890    // `api_retry` NDJSON fragments into `task_runs.error` because the review
2891    // pipeline formatted rate-limit errors with `err.to_string()` instead of
2892    // routing the raw provider message through `summarize_rate_limit_error`.
2893
2894    #[test]
2895    fn format_review_error_summarises_rate_limit_fragment_with_api_retry() {
2896        // Real-world fragment captured from a minimax review run (issue #3149).
2897        // The provider message contains a complete api_retry JSON line so the
2898        // summariser should produce a clean `status=… after … attempts` summary.
2899        let raw_fragment = "{\"attempt\":6,\"max_retries\":10,\"retry_delay_ms\":17266.789,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"5c7148ee\"}\n\
2900                            {\"type\":\"system\",\"subtype\":\"api_retry\",\"attempt\":7,\"max_retries\":10,\"retry_delay_ms\":35162.66,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"5c7148ee\"}";
2901        let err = runner::agents::AgentError::RateLimit {
2902            message: raw_fragment.to_string(),
2903        };
2904
2905        let stored = format_review_error("minimax", &err);
2906
2907        // Must not contain raw JSON braces or the api_retry marker — these are
2908        // exactly the leaks that filled `task_runs.error` with noisy payloads.
2909        assert!(
2910            !stored.contains("{\"type\""),
2911            "stored error must not contain raw NDJSON envelope, got: {stored}"
2912        );
2913        assert!(
2914            !stored.contains("subtype"),
2915            "stored error must not contain raw api_retry fragment, got: {stored}"
2916        );
2917        assert!(
2918            !stored.contains("session_id"),
2919            "stored error must not contain session_id from provider payload, got: {stored}"
2920        );
2921        assert_eq!(
2922            stored,
2923            "minimax rate limit: status=429 after 7 attempts (last delay 35s)"
2924        );
2925    }
2926
2927    #[test]
2928    fn format_review_error_summarises_truncated_fragment_without_subtype_marker() {
2929        // Mid-object fragment that begins after the `subtype` key was cut off — this
2930        // is the shape that landed in `task_runs.error` for rows 13634/13542/9821
2931        // before the fix. The summariser's fallback path should emit a compact
2932        // hint rather than echoing the raw payload.
2933        let raw_fragment = ",\"attempt\":5,\"max_retries\":10,\"retry_delay_ms\":9034.81,\"error_status\":429,\"error\":\"rate_limit\",\"session_id\":\"992e17d3\"";
2934        let err = runner::agents::AgentError::RateLimit {
2935            message: raw_fragment.to_string(),
2936        };
2937
2938        let stored = format_review_error("glm", &err);
2939
2940        assert!(stored.starts_with("glm rate limit:"));
2941        assert!(
2942            !stored.contains("session_id"),
2943            "stored error must not leak session_id from truncated fragment, got: {stored}"
2944        );
2945        // The fragment first-line truncation path is bounded — verify the stored
2946        // string stays well under the 512-char cap suggested by the bug report.
2947        assert!(
2948            stored.len() <= 200,
2949            "stored error should be compact (<=200 chars), got {} chars: {stored}",
2950            stored.len()
2951        );
2952    }
2953
2954    #[test]
2955    fn format_review_error_preserves_non_rate_limit_variants() {
2956        let err = runner::agents::AgentError::Auth {
2957            message: "invalid bearer token".to_string(),
2958        };
2959        assert_eq!(
2960            format_review_error("kimi", &err),
2961            "kimi auth error: invalid bearer token"
2962        );
2963
2964        let err = runner::agents::AgentError::Timeout {
2965            elapsed: std::time::Duration::from_secs(1801),
2966        };
2967        assert_eq!(
2968            format_review_error("kimi", &err),
2969            "kimi timed out after 1801s"
2970        );
2971    }
2972
2973    fn git(dir: &std::path::Path, args: &[&str]) {
2974        let mut cmd = std::process::Command::new("git");
2975        cmd.current_dir(dir);
2976        // Global commit.gpgsign calls the operator's signer (1Password)
2977        if args.first() == Some(&"commit") {
2978            cmd.args(["-c", "commit.gpgsign=false"]);
2979        }
2980        let status = cmd.args(args).status().unwrap();
2981        assert!(status.success(), "git {:?} failed", args);
2982    }
2983
2984    #[test]
2985    fn test_pull_request_review_requests_changes() {
2986        let review = PullRequestReview {
2987            review: GitHubReview {
2988                id: 1,
2989                user: GitHubUser {
2990                    login: "reviewer".to_string(),
2991                },
2992                body: Some("Please fix".to_string()),
2993                state: "CHANGES_REQUESTED".to_string(),
2994                html_url: None,
2995                submitted_at: "2024-01-01T00:00:00Z".to_string(),
2996                commit_id: None,
2997            },
2998            comments: vec![],
2999        };
3000        assert!(review.requests_changes());
3001    }
3002
3003    #[test]
3004    fn test_pull_request_review_does_not_request_changes() {
3005        let review = PullRequestReview {
3006            review: GitHubReview {
3007                id: 1,
3008                user: GitHubUser {
3009                    login: "reviewer".to_string(),
3010                },
3011                body: Some("LGTM".to_string()),
3012                state: "APPROVED".to_string(),
3013                html_url: None,
3014                submitted_at: "2024-01-01T00:00:00Z".to_string(),
3015                commit_id: None,
3016            },
3017            comments: vec![],
3018        };
3019        assert!(!review.requests_changes());
3020    }
3021
3022    #[test]
3023    fn test_pull_request_review_actionable_comments_filters_empty_and_replies() {
3024        let review = PullRequestReview {
3025            review: GitHubReview {
3026                id: 1,
3027                user: GitHubUser {
3028                    login: "reviewer".to_string(),
3029                },
3030                body: None,
3031                state: "CHANGES_REQUESTED".to_string(),
3032                html_url: None,
3033                submitted_at: "2024-01-01T00:00:00Z".to_string(),
3034                commit_id: None,
3035            },
3036            comments: vec![
3037                GitHubReviewComment {
3038                    id: 1,
3039                    user: GitHubUser {
3040                        login: "reviewer".to_string(),
3041                    },
3042                    body: "Fix this issue".to_string(),
3043                    path: "src/main.rs".to_string(),
3044                    line: Some(10),
3045                    original_line: Some(10),
3046                    commit_id: "abc".to_string(),
3047                    original_commit_id: "abc".to_string(),
3048                    html_url: "url".to_string(),
3049                    created_at: "2024-01-01T00:00:00Z".to_string(),
3050                    updated_at: "2024-01-01T00:00:00Z".to_string(),
3051                    in_reply_to_id: None,
3052                    diff_hunk: Some(
3053                        "@@ -8,5 +8,5 @@ fn main() {\n-    let x = 1;\n+    let x = 2;".to_string(),
3054                    ),
3055                },
3056                GitHubReviewComment {
3057                    id: 2,
3058                    user: GitHubUser {
3059                        login: "reviewer".to_string(),
3060                    },
3061                    body: "".to_string(),
3062                    path: "src/lib.rs".to_string(),
3063                    line: Some(20),
3064                    original_line: Some(20),
3065                    commit_id: "abc".to_string(),
3066                    original_commit_id: "abc".to_string(),
3067                    html_url: "url".to_string(),
3068                    created_at: "2024-01-01T00:00:00Z".to_string(),
3069                    updated_at: "2024-01-01T00:00:00Z".to_string(),
3070                    in_reply_to_id: None,
3071                    diff_hunk: None,
3072                },
3073                GitHubReviewComment {
3074                    id: 3,
3075                    user: GitHubUser {
3076                        login: "reviewer".to_string(),
3077                    },
3078                    body: "Reply to this".to_string(),
3079                    path: "src/lib.rs".to_string(),
3080                    line: Some(30),
3081                    original_line: Some(30),
3082                    commit_id: "abc".to_string(),
3083                    original_commit_id: "abc".to_string(),
3084                    html_url: "url".to_string(),
3085                    created_at: "2024-01-01T00:00:00Z".to_string(),
3086                    updated_at: "2024-01-01T00:00:00Z".to_string(),
3087                    in_reply_to_id: Some(1),
3088                    diff_hunk: None,
3089                },
3090            ],
3091        };
3092        let actionable = review.actionable_comments();
3093        assert_eq!(actionable.len(), 1);
3094        assert_eq!(actionable[0].id, 1);
3095        assert_eq!(actionable[0].body, "Fix this issue");
3096        assert_eq!(actionable[0].path, "src/main.rs");
3097        assert_eq!(
3098            actionable[0].diff_hunk.as_ref().unwrap(),
3099            "@@ -8,5 +8,5 @@ fn main() {\n-    let x = 1;\n+    let x = 2;"
3100        );
3101    }
3102
3103    #[test]
3104    fn test_router_config_model_for_complexity_returns_none_without_config() {
3105        // With no model_map configured, model_for_complexity must return None rather than
3106        // silently using a hardcoded model that may not exist for the given agent.
3107        let cfg = RouterConfig::default();
3108        assert!(cfg.model_for_complexity("claude", "simple", "").is_none());
3109        assert!(cfg.model_for_complexity("opencode", "review", "").is_none());
3110        assert!(cfg
3111            .model_for_complexity("unknown_agent_xyz", "simple", "")
3112            .is_none());
3113    }
3114
3115    #[test]
3116    fn review_status_maps_decisions() {
3117        assert_eq!(
3118            review_status(&ReviewDecision::Approve).unwrap().0,
3119            "success"
3120        );
3121        let changes = ReviewDecision::RequestChanges {
3122            notes: String::new(),
3123            issues: vec![],
3124        };
3125        assert_eq!(review_status(&changes).unwrap().0, "failure");
3126        assert!(review_status(&ReviewDecision::Failed("x".into())).is_none());
3127    }
3128
3129    #[test]
3130    fn review_started_comment_includes_agent_and_model() {
3131        let expected_footer = attribution_footer("Review started", "kimi", Some("opus"));
3132        let expected = format!("🔍 Automated review started{}", expected_footer);
3133        assert_eq!(review_started_comment("kimi", "opus"), expected);
3134    }
3135
3136    // ── build_review_context: no worktree/branch/pr (issue #3623) ───────────
3137
3138    fn no_code_review_task(id: &str) -> ExternalTask {
3139        ExternalTask {
3140            id: crate::backends::ExternalId(id.to_string()),
3141            title: "opencode discovery timeout".to_string(),
3142            body: "".to_string(),
3143            state: "open".to_string(),
3144            labels: vec![],
3145            author: "tester".to_string(),
3146            created_at: "2026-01-01T00:00:00Z".to_string(),
3147            updated_at: "2026-01-01T00:00:00Z".to_string(),
3148            url: "https://example.com/issues/1".to_string(),
3149        }
3150    }
3151
3152    #[tokio::test]
3153    async fn build_review_context_reroutes_external_task_with_no_code() {
3154        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3155        let repo = "owner/repo";
3156        let ext = no_code_review_task("3617");
3157        store.ensure_external_task(repo, &ext).await.unwrap();
3158
3159        let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3160        let task_manager = Arc::new(TaskManager::with_store(
3161            backend,
3162            Arc::clone(&store),
3163            repo.to_string(),
3164        ));
3165        let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3166
3167        let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3168            .await
3169            .unwrap();
3170
3171        assert!(
3172            matches!(phase, ReviewPhase::EarlyReturn(ReviewDecision::Rerouted)),
3173            "external task with no worktree/branch/pr must reroute, not skip-to-done"
3174        );
3175
3176        let stored = store
3177            .get_by_external_id(repo, &ext.id.0)
3178            .await
3179            .unwrap()
3180            .expect("task must still exist");
3181        assert_eq!(stored.status, TaskStatus::New);
3182        assert!(stored.agent.is_none());
3183        assert!(stored.model.is_none());
3184    }
3185
3186    #[tokio::test]
3187    async fn build_review_context_blocks_external_task_after_max_reroutes() {
3188        // Isolate from the real ~/.orch/config.yml (which may set a different
3189        // workflow.max_attempts) so the default of 3 reroutes is deterministic.
3190        let temp_home = TempDir::new().unwrap();
3191        let orch_dir = temp_home.path().join(".orch");
3192        std::fs::create_dir_all(&orch_dir).unwrap();
3193        let old_orch_home = std::env::var_os("ORCH_HOME");
3194        std::env::set_var("ORCH_HOME", &orch_dir);
3195
3196        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3197        let repo = "owner/repo";
3198        let ext = no_code_review_task("3618");
3199        store.ensure_external_task(repo, &ext).await.unwrap();
3200
3201        let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3202        let task_manager = Arc::new(TaskManager::with_store(
3203            backend,
3204            Arc::clone(&store),
3205            repo.to_string(),
3206        ));
3207        let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3208
3209        // Default max_reroute_attempts is 3 — exhaust it.
3210        for _ in 0..3 {
3211            let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3212                .await
3213                .unwrap();
3214            if let ReviewPhase::EarlyReturn(ReviewDecision::Blocked(_)) = phase {
3215                break;
3216            }
3217        }
3218
3219        let stored = store
3220            .get_by_external_id(repo, &ext.id.0)
3221            .await
3222            .unwrap()
3223            .expect("task must still exist");
3224        assert_eq!(stored.status, TaskStatus::Blocked);
3225
3226        if let Some(old) = old_orch_home {
3227            std::env::set_var("ORCH_HOME", old);
3228        } else {
3229            std::env::remove_var("ORCH_HOME");
3230        }
3231    }
3232
3233    #[tokio::test]
3234    async fn build_review_context_marks_internal_task_done_with_no_code() {
3235        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3236        let repo = "owner/repo";
3237        let internal_id = store
3238            .create_internal(repo, "Daily retrospective", "", "job", "job-1", None)
3239            .await
3240            .unwrap();
3241        let ext_id = format!("internal:{internal_id}");
3242        let ext = no_code_review_task(&ext_id);
3243
3244        let backend = Arc::new(crate::backends::test_helpers::NoopBackend);
3245        let task_manager = Arc::new(TaskManager::with_store(
3246            backend,
3247            Arc::clone(&store),
3248            repo.to_string(),
3249        ));
3250        let router = Arc::new(RwLock::new(Router::new(RouterConfig::default())));
3251
3252        let phase = build_review_context(&ext, repo, &router, &task_manager, &store)
3253            .await
3254            .unwrap();
3255
3256        assert!(matches!(
3257            phase,
3258            ReviewPhase::EarlyReturn(ReviewDecision::Skipped)
3259        ));
3260        let stored = store.get(internal_id).await.unwrap();
3261        assert_eq!(stored.status, TaskStatus::Done);
3262    }
3263
3264    #[test]
3265    fn no_code_reroute_skip_check_rejects_agent_stream_disconnects() {
3266        let last_error =
3267            "codex failed: Reconnecting... (stream disconnected before completion: Broken pipe)";
3268        assert!(!should_skip_no_code_reroute_increment(last_error));
3269    }
3270
3271    // ── review fallback rate-limit/auth scan is bounded, not full-text ──────
3272    //
3273    // Regression test for a review transcript whose unparseable fallback text
3274    // is a long report/diff quoting a status table (`outcome` column values
3275    // like `rate_limit`, `failed`, `timeout`). Scanning the full text falsely
3276    // classified the review run as rate-limited even though the review agent
3277    // never actually hit a rate limit.
3278
3279    #[test]
3280    fn review_fallback_scan_ignores_rate_limit_word_buried_in_long_report() {
3281        let padding = "x".repeat(10_000);
3282        let text_for_review = format!(
3283            "{padding}\n+| codex | gpt-5.4 | failed | 1 |\n+| codex | gpt-5.5 | failed / aborted / (empty) | 1 each |\n+| kimi | opus | rate_limit | 1 |\n{padding}"
3284        );
3285
3286        let scan_head = runner::agents::patterns::safe_head(
3287            &text_for_review,
3288            runner::agents::patterns::RATE_LIMIT_SCAN_HEAD_BYTES,
3289        );
3290        let scan_tail = runner::agents::patterns::safe_tail(
3291            &text_for_review,
3292            runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3293        );
3294
3295        assert!(runner::agents::patterns::detect_rate_limit(scan_head).is_none());
3296        assert!(runner::agents::patterns::detect_rate_limit(scan_tail).is_none());
3297    }
3298
3299    #[test]
3300    fn review_fallback_scan_still_detects_real_rate_limit_at_tail() {
3301        let padding = "x".repeat(10_000);
3302        let text_for_review =
3303            format!("{padding}\nError: rate limit exceeded, please try again later");
3304
3305        let scan_tail = runner::agents::patterns::safe_tail(
3306            &text_for_review,
3307            runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3308        );
3309
3310        assert!(runner::agents::patterns::detect_rate_limit(scan_tail).is_some());
3311    }
3312
3313    #[tokio::test]
3314    async fn recover_missing_review_worktree_recreates_worktree() {
3315        let temp_home = TempDir::new().unwrap();
3316        let orch_dir = temp_home.path().join(".orch");
3317        std::fs::create_dir_all(&orch_dir).unwrap();
3318        let old_orch_home = std::env::var_os("ORCH_HOME");
3319        std::env::set_var("ORCH_HOME", &orch_dir);
3320
3321        let project_dir = temp_home.path().join("project");
3322        std::fs::create_dir_all(&project_dir).unwrap();
3323        git(&project_dir, &["init", "-b", "main"]);
3324        git(&project_dir, &["config", "user.name", "Test User"]);
3325        git(&project_dir, &["config", "user.email", "test@example.com"]);
3326        std::fs::write(project_dir.join("README.md"), "hello").unwrap();
3327        git(&project_dir, &["add", "README.md"]);
3328        git(&project_dir, &["commit", "-m", "init"]);
3329
3330        std::fs::write(
3331            orch_dir.join("config.yml"),
3332            format!("projects:\n  - {}\n", project_dir.display()),
3333        )
3334        .unwrap();
3335        std::fs::write(project_dir.join(".orch.yml"), "gh:\n  repo: owner/repo\n").unwrap();
3336
3337        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3338        let repo = "owner/repo";
3339        store
3340            .create_internal(repo, "Fix review recovery", "", "review", "review-1", None)
3341            .await
3342            .unwrap();
3343        let ext = ExternalTask {
3344            id: crate::backends::ExternalId("1".to_string()),
3345            title: "Fix review recovery".to_string(),
3346            body: "".to_string(),
3347            state: "open".to_string(),
3348            labels: vec![],
3349            author: "tester".to_string(),
3350            created_at: "2026-01-01T00:00:00Z".to_string(),
3351            updated_at: "2026-01-01T00:00:00Z".to_string(),
3352            url: "https://example.com/issues/1".to_string(),
3353        };
3354        store.ensure_external_task(repo, &ext).await.unwrap();
3355
3356        let recovered = recover_missing_review_worktree(&ext, repo, &store)
3357            .await
3358            .unwrap();
3359
3360        assert!(recovered.work_dir.exists());
3361        assert!(recovered.branch.starts_with("gh-issue-1-"));
3362
3363        if let Some(old) = old_orch_home {
3364            std::env::set_var("ORCH_HOME", old);
3365        } else {
3366            std::env::remove_var("ORCH_HOME");
3367        }
3368    }
3369
3370    // ── count_ahead_commits ─────────────────────────────────────────────────
3371
3372    #[tokio::test]
3373    async fn count_ahead_commits_returns_zero_when_no_ahead_commits() {
3374        let temp = TempDir::new().unwrap();
3375        let dir = temp.path();
3376
3377        // Create a bare repo to act as the 'origin' remote
3378        let bare = temp.path().join("bare.git");
3379        std::fs::create_dir_all(&bare).unwrap();
3380        std::process::Command::new("git")
3381            .args(["init", "--bare"])
3382            .current_dir(&bare)
3383            .output()
3384            .unwrap();
3385
3386        git(dir, &["init", "-b", "main"]);
3387        git(dir, &["config", "user.name", "Test"]);
3388        git(dir, &["config", "user.email", "test@test.com"]);
3389        std::fs::write(dir.join("f"), "a").unwrap();
3390        git(dir, &["add", "f"]);
3391        git(dir, &["commit", "-m", "init"]);
3392        std::process::Command::new("git")
3393            .args(["remote", "add", "origin", bare.to_str().unwrap()])
3394            .current_dir(dir)
3395            .output()
3396            .unwrap();
3397        std::process::Command::new("git")
3398            .args(["push", "origin", "main"])
3399            .current_dir(dir)
3400            .output()
3401            .unwrap();
3402
3403        // origin/main now exists and HEAD is at the same commit — zero ahead
3404        let result = count_ahead_commits(dir, "main").await;
3405        assert!(
3406            result.is_ok(),
3407            "expected Ok when origin/main exists, got {result:?}"
3408        );
3409        assert_eq!(
3410            result.unwrap(),
3411            0,
3412            "expected 0 commits ahead when branch matches origin/main"
3413        );
3414    }
3415
3416    #[tokio::test]
3417    async fn count_ahead_commits_returns_error_when_remote_ref_missing() {
3418        let temp = TempDir::new().unwrap();
3419        let dir = temp.path();
3420        git(dir, &["init", "-b", "main"]);
3421        git(dir, &["config", "user.name", "Test"]);
3422        git(dir, &["config", "user.email", "test@test.com"]);
3423        std::fs::write(dir.join("f"), "a").unwrap();
3424        git(dir, &["add", "f"]);
3425        git(dir, &["commit", "-m", "init"]);
3426        // No remote 'origin' configured — rev-list will fail
3427        let result = count_ahead_commits(dir, "main").await;
3428        assert!(
3429            result.is_err(),
3430            "expected Err when origin/main does not exist, got {result:?}"
3431        );
3432        let err = result.unwrap_err();
3433        assert!(
3434            err.contains("git rev-list exited"),
3435            "error should mention exit failure, got: {err}"
3436        );
3437    }
3438
3439    #[tokio::test]
3440    async fn count_ahead_commits_returns_error_when_git_cannot_start() {
3441        // Use a path that is not a git repo — git will fail with non-zero exit
3442        let temp = TempDir::new().unwrap();
3443        let dir = temp.path();
3444        // Not a git repo at all
3445        let result = count_ahead_commits(dir, "main").await;
3446        assert!(
3447            result.is_err(),
3448            "expected Err when directory is not a git repo, got {result:?}"
3449        );
3450    }
3451
3452    // ── review cooldown escalation (issue #3314) ────────────────────────────
3453
3454    /// Review `InvalidResponse` (hard failure: empty/error output) must apply the
3455    /// persistent model backoff (~4h base) so the model isn't re-selected on the
3456    /// next tick after a brief standard cooldown expires.
3457    ///
3458    /// Regression test for issue #3314: `record_model_failure` (5-min base) was used
3459    /// instead of `record_persistent_model_failure` (4h base), allowing repeated
3460    /// re-selection of a structurally broken review model.
3461    #[serial_test::serial(cooldown_state)]
3462    #[tokio::test]
3463    async fn review_invalid_response_applies_persistent_model_cooldown() {
3464        crate::engine::cooldown::reset_global_state().await;
3465        let agent = "opencode-3314-review-invalid";
3466        let model = "opencode/north-mini-code-free";
3467        let key = format!("{agent}:{model}");
3468
3469        assert!(
3470            !crate::engine::cooldown::is_model_in_cooldown(agent, model),
3471            "model should not be in cooldown before record_review_agent_failure"
3472        );
3473
3474        let err = runner::agents::AgentError::InvalidResponse {
3475            raw: "empty review output".to_string(),
3476        };
3477        record_review_agent_failure("test-3314-a", agent, Some(model), &err).await;
3478
3479        let now = chrono::Utc::now().timestamp();
3480        let until = crate::engine::cooldown::cooldown_until(&key)
3481            .expect("review InvalidResponse should set model cooldown");
3482        let remaining = until.saturating_sub(now);
3483
3484        // Persistent backoff base is ~4h (14400s); standard model backoff base is 5min (300s).
3485        // Verify the persistent (4h) path was taken, not the short one.
3486        assert!(
3487            remaining >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3488            "review InvalidResponse must apply ~4h persistent cooldown (issue #3314), \
3489             got {remaining}s — may have used 5-min standard backoff instead"
3490        );
3491    }
3492
3493    /// Review `parse_error` (got output but couldn't parse it as a review response) must
3494    /// also apply the persistent model backoff so the model is cooled long enough to
3495    /// prevent re-selection after a brief cooldown or a success-reset from another task.
3496    ///
3497    /// Verifies the `record_review_agent_failure` path for `InvalidResponse`
3498    /// (which covers both the hard-failure and soft parse-error paths that were
3499    /// changed in the fix for issue #3314).
3500    #[serial_test::serial(cooldown_state)]
3501    #[tokio::test]
3502    async fn review_persistent_cooldown_survives_after_first_failure() {
3503        crate::engine::cooldown::reset_global_state().await;
3504        let agent = "opencode-3314-review-parse";
3505        let model = "opencode/north-mini-code-free";
3506        let key = format!("{agent}:{model}");
3507
3508        // First failure: persistent backoff should apply immediately
3509        let err = runner::agents::AgentError::InvalidResponse {
3510            raw: "failed to parse review response".to_string(),
3511        };
3512        record_review_agent_failure("test-3314-b", agent, Some(model), &err).await;
3513
3514        let now = chrono::Utc::now().timestamp();
3515        let until1 = crate::engine::cooldown::cooldown_until(&key)
3516            .expect("first failure should set cooldown");
3517        let remaining1 = until1.saturating_sub(now);
3518
3519        // Simulate a success for the same agent+model (from a non-review task).
3520        // This resets failure_count:{agent}:{model} to 0.
3521        crate::engine::cooldown::record_agent_success(agent, model).await;
3522
3523        // After success reset, cooldown timestamp is still set — the success only
3524        // resets the failure count, NOT the existing cooldown timer.
3525        // The next review failure will restart from count=1 (base backoff).
3526        // The key invariant: persistent base (4h) >> standard base (5min) so even
3527        // after a reset, the first-failure cooldown still lasts ~4h.
3528        assert!(
3529            remaining1 >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3530            "after success reset, first-failure cooldown should still be ~4h, got {remaining1}s"
3531        );
3532    }
3533
3534    /// Regression test for issue #3551: a billing-cycle-exhaustion message caught by
3535    /// the review module's *inline text scan* (`detect_rate_limit` on raw review
3536    /// output, used when the review process exits without a clean `AgentError`) must
3537    /// get the same treatment as the `AgentError`-based failure path fixed by #3530 —
3538    /// not a hardcoded `"rate_limit"` outcome plus a generic short agent cooldown.
3539    #[serial_test::serial(cooldown_state)]
3540    #[tokio::test]
3541    async fn review_inline_scan_billing_cycle_message_gets_model_scoped_cooldown() {
3542        crate::engine::cooldown::reset_global_state().await;
3543        let agent = "kimi-3551-review-inline";
3544        let model = "opus";
3545        let key = format!("{agent}:{model}");
3546
3547        // Verbatim text from the observed kimi failure (issue #3551).
3548        let text_for_review = "kimi rate limit: Failed to authenticate. API Error: 403 \
3549            You've reached your usage limit for this billing cycle. Your quota will be \
3550            refreshed next cycle.";
3551
3552        let scan_tail = runner::agents::patterns::safe_tail(
3553            text_for_review,
3554            runner::agents::patterns::RATE_LIMIT_SCAN_TAIL_BYTES,
3555        );
3556        let err = runner::agents::patterns::detect_rate_limit(scan_tail)
3557            .expect("billing-cycle message must still be caught by the rate-limit scan");
3558
3559        assert_eq!(
3560            outcome_for_agent_error(&err),
3561            "billing_cycle_exhausted",
3562            "inline-scanned billing-cycle message must not be stored as plain rate_limit"
3563        );
3564
3565        record_review_agent_failure("test-3551", agent, Some(model), &err).await;
3566
3567        let now = chrono::Utc::now().timestamp();
3568        let until = crate::engine::cooldown::cooldown_until(&key)
3569            .expect("billing-cycle exhaustion must set a model-scoped cooldown");
3570        let remaining = until.saturating_sub(now);
3571        // Known model + BillingCycleExhausted routes to record_persistent_model_failure
3572        // (~4h base, 7d cap), not the agent-wide record_credit_exhaustion (~24h) path —
3573        // the quota is scoped to this model, so unrelated models on the same agent must
3574        // stay routable.
3575        assert!(
3576            remaining >= crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS - 30,
3577            "billing-cycle exhaustion for a known model must apply the ~4h persistent \
3578             model cooldown, got {remaining}s — may have used the generic short agent \
3579             cooldown instead"
3580        );
3581    }
3582
3583    // ── review success resets failure_count (issue #3478) ──────────────────
3584
3585    fn test_review_task(id: &str) -> ExternalTask {
3586        ExternalTask {
3587            id: crate::backends::ExternalId(id.to_string()),
3588            title: "Test review task".to_string(),
3589            body: "".to_string(),
3590            state: "open".to_string(),
3591            labels: vec![],
3592            author: "tester".to_string(),
3593            created_at: "2026-01-01T00:00:00Z".to_string(),
3594            updated_at: "2026-01-01T00:00:00Z".to_string(),
3595            url: "https://example.com/issues/1".to_string(),
3596        }
3597    }
3598
3599    fn test_review_context(agent: &str, model: &str, work_dir: &std::path::Path) -> ReviewContext {
3600        ReviewContext {
3601            store_id: None,
3602            had_prev_push_failure: false,
3603            review_attempt: 1,
3604            worktree_path: work_dir.to_path_buf(),
3605            branch_name: "gh-issue-1-test".to_string(),
3606            pr_number: 1,
3607            head_sha: None,
3608            default_branch: "main".to_string(),
3609            review_agent: agent.to_string(),
3610            review_model: Some(model.to_string()),
3611            review_task_id: "review-1".to_string(),
3612            review_attempt_dir: work_dir.to_path_buf(),
3613            output_file: work_dir.join("output.txt"),
3614            invocation: runner::agent::AgentInvocation {
3615                agent: agent.to_string(),
3616                model: Some(model.to_string()),
3617                work_dir: work_dir.to_path_buf(),
3618                system_prompt: String::new(),
3619                agent_message: String::new(),
3620                task_id: "review-1".to_string(),
3621                disallowed_tools: vec![],
3622                git_author_name: "Test".to_string(),
3623                git_author_email: "test@example.com".to_string(),
3624                output_file: work_dir.join("output.txt"),
3625                timeout_seconds: 0,
3626                repo: "owner/repo".to_string(),
3627                attempt: 1,
3628            },
3629            same_agent_fallback: false,
3630        }
3631    }
3632
3633    fn test_parsed_review(decision: ReviewDecision) -> ParsedReview {
3634        ParsedReview {
3635            run_id: None,
3636            exit_code: 0,
3637            stderr: String::new(),
3638            raw_output: String::new(),
3639            text_for_review: String::new(),
3640            token_usage: RunTokenUsage::default(),
3641            review_notes_for_comment: String::new(),
3642            decision,
3643        }
3644    }
3645
3646    fn write_review_output(dir: &std::path::Path, stdout: &str, exit_code: i32) {
3647        std::fs::write(dir.join("output.txt"), stdout).unwrap();
3648        std::fs::write(dir.join("exit.txt"), exit_code.to_string()).unwrap();
3649        std::fs::write(dir.join("stderr.txt"), "").unwrap();
3650    }
3651
3652    /// A kimi review that exits mid-thought (no result envelope, last content is a
3653    /// thinking block) must be stored as incomplete and must not apply the 4h
3654    /// persistent model cooldown (issue #3734).
3655    #[serial_test::serial(cooldown_state)]
3656    #[tokio::test]
3657    async fn parse_review_output_mid_thought_is_incomplete_without_persistent_cooldown() {
3658        crate::engine::cooldown::reset_global_state().await;
3659        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3660        let work_dir = TempDir::new().unwrap();
3661        let agent = "kimi";
3662        let model = "opus-3734-mid-thought";
3663        let ctx = test_review_context(agent, model, work_dir.path());
3664        let redraw = "^D\u{0008}".repeat(20);
3665        let ndjson = format!(
3666            "{}\n{}\n{}\n{}\n{}",
3667            r#"{"type":"system","subtype":"thinking_tokens","tokens":1945}"#,
3668            r#"{"type":"assistant","message":{"content":[{"type":"text","text":"CI passes. Now verifying callers and the lease logic edge cases:"}],"stop_reason":null,"usage":{"input_tokens":10,"output_tokens":0}}}"#,
3669            r#"{"type":"assistant","message":{"content":[{"type":"thinking","thinking":"checking lease edge cases"}],"stop_reason":null,"usage":{"input_tokens":10,"output_tokens":0}}}"#,
3670            "\x1b[?25h",
3671            redraw,
3672        );
3673        write_review_output(work_dir.path(), &ndjson, 0);
3674
3675        let task = test_review_task("3734");
3676        let phase = parse_review_output(
3677            &task,
3678            "owner/repo",
3679            &ctx,
3680            ReviewRun { run_id: None },
3681            &store,
3682        )
3683        .await;
3684
3685        match phase {
3686            ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3687                assert!(
3688                    reason.contains("incomplete:"),
3689                    "mid-thought review must be stored as incomplete, got {reason}"
3690                );
3691            }
3692            ReviewPhase::EarlyReturn(_) => panic!("expected Failed(incomplete)"),
3693            ReviewPhase::Ready(_) => panic!("expected Failed(incomplete), got Ready"),
3694        }
3695        assert!(
3696            !crate::engine::cooldown::is_model_in_cooldown(agent, model),
3697            "mid-thought incomplete must not apply persistent model cooldown"
3698        );
3699    }
3700
3701    /// A tool_use as the last assistant content with no result envelope is the
3702    /// same incomplete shape as a thinking block (issue #3734).
3703    #[serial_test::serial(cooldown_state)]
3704    #[tokio::test]
3705    async fn parse_review_output_tool_use_without_envelope_is_incomplete() {
3706        crate::engine::cooldown::reset_global_state().await;
3707        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3708        let work_dir = TempDir::new().unwrap();
3709        let agent = "kimi";
3710        let model = "opus-3734-tool-use";
3711        let ctx = test_review_context(agent, model, work_dir.path());
3712        let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Read","input":{"file_path":"src/lib.rs"}}]}}"#;
3713        write_review_output(work_dir.path(), ndjson, 0);
3714
3715        let task = test_review_task("3734-tool");
3716        let phase = parse_review_output(
3717            &task,
3718            "owner/repo",
3719            &ctx,
3720            ReviewRun { run_id: None },
3721            &store,
3722        )
3723        .await;
3724
3725        match phase {
3726            ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3727                assert!(
3728                    reason.contains("incomplete:"),
3729                    "tool_use without envelope must be incomplete, got {reason}"
3730                );
3731            }
3732            ReviewPhase::EarlyReturn(_) => panic!("expected Failed(incomplete)"),
3733            ReviewPhase::Ready(_) => panic!("expected Failed(incomplete), got Ready"),
3734        }
3735        assert!(!crate::engine::cooldown::is_model_in_cooldown(agent, model));
3736    }
3737
3738    /// A finished result envelope whose text is not a review decision still
3739    /// gets the persistent format-failure cooldown.
3740    #[serial_test::serial(cooldown_state)]
3741    #[tokio::test]
3742    async fn parse_review_output_unparseable_envelope_applies_persistent_cooldown() {
3743        crate::engine::cooldown::reset_global_state().await;
3744        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3745        let work_dir = TempDir::new().unwrap();
3746        let agent = "kimi";
3747        let model = "opus-3734-format";
3748        let ctx = test_review_context(agent, model, work_dir.path());
3749        let ndjson = concat!(
3750            r#"{"type":"assistant","message":{"content":[{"type":"text","text":"I have finished looking at the code."}]}}"#,
3751            "\n",
3752            r#"{"type":"result","subtype":"success","is_error":false,"result":"I have finished looking at the code."}"#,
3753        );
3754        write_review_output(work_dir.path(), ndjson, 0);
3755
3756        let task = test_review_task("3734-format");
3757        let phase = parse_review_output(
3758            &task,
3759            "owner/repo",
3760            &ctx,
3761            ReviewRun { run_id: None },
3762            &store,
3763        )
3764        .await;
3765
3766        match phase {
3767            ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3768                assert!(
3769                    reason.contains("parse error"),
3770                    "finished unparseable envelope must stay a parse error, got {reason}"
3771                );
3772            }
3773            ReviewPhase::EarlyReturn(_) => panic!("expected Failed(parse error)"),
3774            ReviewPhase::Ready(_) => panic!("expected Failed(parse error), got Ready"),
3775        }
3776        assert!(
3777            crate::engine::cooldown::is_model_in_cooldown(agent, model),
3778            "finished format failure must apply persistent model cooldown"
3779        );
3780    }
3781
3782    /// Missing result envelope with a review decision in assistant text uses
3783    /// the assistant-text extractor, not MiniMax status=done synthesis.
3784    #[serial_test::serial(cooldown_state)]
3785    #[tokio::test]
3786    async fn parse_review_output_missing_envelope_uses_assistant_decision() {
3787        crate::engine::cooldown::reset_global_state().await;
3788        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3789        let work_dir = TempDir::new().unwrap();
3790        let agent = "kimi";
3791        let model = "opus-3734-assistant";
3792        let ctx = test_review_context(agent, model, work_dir.path());
3793        let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"text","text":"{\"decision\":\"approve\",\"notes\":\"LGTM\",\"issues\":[]}"}]}}"#;
3794        write_review_output(work_dir.path(), ndjson, 0);
3795
3796        let task = test_review_task("3734-asst");
3797        let phase = parse_review_output(
3798            &task,
3799            "owner/repo",
3800            &ctx,
3801            ReviewRun { run_id: None },
3802            &store,
3803        )
3804        .await;
3805
3806        match phase {
3807            ReviewPhase::Ready(parsed) => {
3808                assert!(matches!(parsed.decision, ReviewDecision::Approve));
3809            }
3810            ReviewPhase::EarlyReturn(_) => {
3811                panic!("expected Ready(Approve) from assistant text, got EarlyReturn")
3812            }
3813        }
3814        assert!(!crate::engine::cooldown::is_model_in_cooldown(agent, model));
3815    }
3816
3817    /// MiniMax/Kimi parse_response synthesizes status=done from any assistant
3818    /// text. That task-path fallback must not be treated as a review approval.
3819    #[serial_test::serial(cooldown_state)]
3820    #[tokio::test]
3821    async fn parse_review_output_does_not_treat_task_done_synthesis_as_approval() {
3822        crate::engine::cooldown::reset_global_state().await;
3823        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3824        let work_dir = TempDir::new().unwrap();
3825        let agent = "kimi";
3826        let model = "opus-3734-done";
3827        let ctx = test_review_context(agent, model, work_dir.path());
3828        let ndjson = r#"{"type":"assistant","message":{"content":[{"type":"text","text":"analyzing the diff"}]}}"#;
3829        write_review_output(work_dir.path(), ndjson, 0);
3830
3831        let task = test_review_task("3734-done");
3832        let phase = parse_review_output(
3833            &task,
3834            "owner/repo",
3835            &ctx,
3836            ReviewRun { run_id: None },
3837            &store,
3838        )
3839        .await;
3840
3841        match phase {
3842            ReviewPhase::EarlyReturn(ReviewDecision::Failed(reason)) => {
3843                assert!(
3844                    reason.contains("parse error"),
3845                    "unfinished prose without a decision must not approve, got {reason}"
3846                );
3847            }
3848            ReviewPhase::Ready(_) => {
3849                panic!("task-path status=done synthesis must not become a review approval")
3850            }
3851            ReviewPhase::EarlyReturn(_) => panic!("expected Failed(parse error)"),
3852        }
3853    }
3854
3855    /// Regression test for issue #3478: a successful review run must reset
3856    /// `failure_count:{agent}:{model}`, mirroring the runner's task-agent
3857    /// success path, for generic/transient failures (not the persistent-model
3858    /// path — see issue #3571, which deliberately gives persistent failures
3859    /// their own counter that a success does NOT reset).
3860    #[serial_test::serial(cooldown_state)]
3861    #[tokio::test]
3862    async fn finalize_review_run_success_resets_generic_model_failure_count() {
3863        crate::engine::cooldown::reset_global_state().await;
3864        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3865        crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3866
3867        let agent = "test-3478-agent";
3868        let model = "test-3478-model";
3869        let failure_key = format!("failure_count:{agent}:{model}");
3870
3871        // AgentFailed is a generic/transient error, routed to record_model_failure
3872        // (not record_persistent_model_failure), so it should still be reset by
3873        // an intervening success.
3874        let err = runner::agents::AgentError::AgentFailed {
3875            message: "agent reported a generic failure".to_string(),
3876        };
3877        record_review_agent_failure("task-3478-a", agent, Some(model), &err).await;
3878        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3879        assert_eq!(
3880            store.kv_get(&failure_key).await.unwrap().as_deref(),
3881            Some("1"),
3882            "sanity check: failure should have incremented the persisted count"
3883        );
3884
3885        let work_dir = TempDir::new().unwrap();
3886        let ctx = test_review_context(agent, model, work_dir.path());
3887        let parsed = test_parsed_review(ReviewDecision::Approve);
3888        let task = test_review_task("1");
3889
3890        finalize_review_run(&task, &ctx, &parsed, &store).await;
3891        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3892
3893        assert_eq!(
3894            store.kv_get(&failure_key).await.unwrap().as_deref(),
3895            Some("0"),
3896            "successful review must reset failure_count:{{agent}}:{{model}} (issue #3478)"
3897        );
3898    }
3899
3900    /// Regression test for issue #3478: successful reviews interleaved between
3901    /// isolated generic failures must keep each failure's backoff at the base
3902    /// duration instead of escalating, since the previous success reset the
3903    /// counter. Uses AgentFailed (generic/transient), not InvalidResponse
3904    /// (persistent-model path — see issue #3571).
3905    #[serial_test::serial(cooldown_state)]
3906    #[tokio::test]
3907    async fn finalize_review_run_interleaved_success_keeps_generic_cooldown_at_base() {
3908        crate::engine::cooldown::reset_global_state().await;
3909        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3910        crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3911
3912        let agent = "test-3478-interleave-agent";
3913        let model = "test-3478-interleave-model";
3914        let cooldown_key = format!("{agent}:{model}");
3915
3916        let work_dir = TempDir::new().unwrap();
3917        let ctx = test_review_context(agent, model, work_dir.path());
3918        let task = test_review_task("2");
3919
3920        for i in 0..4 {
3921            let err = runner::agents::AgentError::AgentFailed {
3922                message: format!("failure {i}"),
3923            };
3924            record_review_agent_failure("task-3478-b", agent, Some(model), &err).await;
3925
3926            let now = chrono::Utc::now().timestamp();
3927            let until = crate::engine::cooldown::cooldown_until(&cooldown_key)
3928                .expect("failure should set cooldown");
3929            let remaining = until.saturating_sub(now);
3930
3931            assert!(
3932                remaining <= crate::engine::cooldown::BACKOFF_BASE_SECS + 60,
3933                "failure #{} must stay at base backoff since the prior success reset the \
3934                 counter (issue #3478); got {remaining}s, base is {}s — an escalating value \
3935                 here means review successes are not resetting failure_count",
3936                i + 1,
3937                crate::engine::cooldown::BACKOFF_BASE_SECS
3938            );
3939
3940            // A successful review interleaves between each failure, mirroring the
3941            // real-world pattern from issue #3478 (reviews for other tasks succeed
3942            // in between isolated failures for this model).
3943            let parsed = test_parsed_review(ReviewDecision::Approve);
3944            finalize_review_run(&task, &ctx, &parsed, &store).await;
3945            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3946        }
3947    }
3948
3949    /// Regression test for issue #3571: unlike generic failures, persistent
3950    /// model failures (e.g. review parse errors, InvalidResponse) must NOT be
3951    /// reset by an intervening successful review. Otherwise a model that fails
3952    /// structurally every so often — but succeeds most of the time — never
3953    /// escalates past the 4h base cooldown and stays in rotation indefinitely.
3954    #[serial_test::serial(cooldown_state)]
3955    #[tokio::test]
3956    async fn finalize_review_run_success_does_not_reset_persistent_model_failure_count() {
3957        crate::engine::cooldown::reset_global_state().await;
3958        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3959        crate::engine::cooldown::init_cooldown_store(store.clone()).await;
3960
3961        let agent = "test-3571-agent";
3962        let model = "test-3571-model";
3963        let cooldown_key = format!("{agent}:{model}");
3964
3965        let work_dir = TempDir::new().unwrap();
3966        let ctx = test_review_context(agent, model, work_dir.path());
3967        let task = test_review_task("3");
3968
3969        // First structural (parse-error-equivalent) failure -> base 4h cooldown.
3970        let err = runner::agents::AgentError::InvalidResponse {
3971            raw: "empty review output".to_string(),
3972        };
3973        record_review_agent_failure("task-3571-a", agent, Some(model), &err).await;
3974        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3975
3976        // A successful review for the same model interleaves, exactly like the
3977        // real-world pattern from issue #3571.
3978        let parsed = test_parsed_review(ReviewDecision::Approve);
3979        finalize_review_run(&task, &ctx, &parsed, &store).await;
3980        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3981
3982        // Second structural failure must escalate from count 2 (~12h), not
3983        // reset back to the 4h base, despite the intervening success.
3984        let err2 = runner::agents::AgentError::InvalidResponse {
3985            raw: "empty review output again".to_string(),
3986        };
3987        record_review_agent_failure("task-3571-b", agent, Some(model), &err2).await;
3988        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
3989
3990        let now = chrono::Utc::now().timestamp();
3991        let until = crate::engine::cooldown::cooldown_until(&cooldown_key)
3992            .expect("second failure should set cooldown");
3993        let remaining = until.saturating_sub(now);
3994
3995        assert!(
3996            remaining > crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS,
3997            "second persistent model failure should escalate past the 4h base despite an \
3998             intervening success (issue #3571); got {remaining}s, base is {}s",
3999            crate::engine::cooldown::PERSISTENT_MODEL_BACKOFF_BASE_SECS
4000        );
4001    }
4002}