Skip to main content

orch/engine/runner/
mod.rs

1//! Task runner — executes tasks using AI agents.
2//!
3//! Replaces `run_task.sh` with pure Rust. The runner:
4//! 1. Sets up a git worktree for isolation
5//! 2. Builds context (project instructions, skills, comments)
6//! 3. Renders prompts and spawns the agent in tmux
7//! 4. Waits for completion, collects and classifies the response
8//! 5. Handles errors (reroute on limits, fallback agents)
9//! 6. Auto-commits, pushes, and creates PRs
10//! 7. Updates status labels and posts result comments
11//!
12//! Submodules handle the heavy lifting:
13//! - [`task_init`] — guard checks, worktree setup, and invocation building
14//! - [`session`] — tmux session lifecycle and output collection
15//! - [`response_handler`] — success path: commit, push, PR, token storage
16//! - [`fallback`] — error classification and recovery strategies
17
18pub mod agent;
19pub mod agents;
20pub mod commit;
21pub mod context;
22pub mod cow;
23pub mod diff;
24pub mod direct;
25pub mod fallback;
26pub mod git_ops;
27#[cfg(test)]
28mod recorded_fixtures;
29pub mod response;
30pub mod response_handler;
31pub mod sandbox;
32pub mod screenshots;
33pub mod session;
34pub mod task_init;
35pub mod worktree;
36
37use crate::backends::{ExternalBackend, ExternalTask, Status};
38use crate::config;
39use crate::engine;
40use crate::engine::cleanup;
41use crate::engine::cooldown;
42use crate::engine::router::RouteResult;
43use crate::engine::tasks;
44use crate::engine::tasks::is_internal_id;
45#[cfg(not(test))]
46use crate::github::http;
47use crate::github::http::GhHttp;
48use crate::home;
49use crate::parser;
50use crate::parser::AgentResponse;
51use crate::parser::Delegation;
52use crate::status_heuristics;
53use crate::store;
54use crate::store::InsertTaskMetric;
55use crate::store::RunTokenUsage;
56use crate::store::TaskStatus;
57use crate::store::TaskStore;
58use crate::tmux::TmuxManager;
59use anyhow::Context;
60use chrono::Utc;
61pub use response::WeightSignal;
62use serde_json::Value;
63use std::collections::HashSet;
64use std::env;
65use std::error;
66use std::fmt;
67use std::fmt::Debug;
68use std::fmt::Formatter;
69use std::path::PathBuf;
70use std::slice;
71use std::sync::Arc;
72use tokio::fs;
73use tokio::task;
74
75/// Fully materialized task-run audit data.
76#[derive(Debug, Clone)]
77pub struct RunAudit {
78    pub stdout: String,
79    pub stderr: String,
80    pub parsed_response: String,
81    pub outcome: String,
82    pub error: String,
83    pub input_tokens: u64,
84    pub output_tokens: u64,
85    pub total_cost_usd: f64,
86    pub duration_secs: f64,
87}
88
89/// Result returned by `run()` so callers can persist the audit trail.
90#[derive(Debug, Clone)]
91pub struct RunExecution {
92    pub status: String,
93    pub exit_code: Option<i32>,
94    pub audit: RunAudit,
95}
96
97fn serialize_parsed_response(
98    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
99    raw_stdout: &str,
100) -> String {
101    match parse_result {
102        Ok(parsed) => serde_json::json!({
103            "response": parsed.response,
104            "input_tokens": parsed.input_tokens,
105            "output_tokens": parsed.output_tokens,
106            "cost_usd": parsed.cost_usd,
107            "duration_ms": parsed.duration_ms,
108        })
109        .to_string(),
110        Err(agents::AgentError::InvalidResponse { raw }) => {
111            serde_json::json!({"raw": raw}).to_string()
112        }
113        Err(_) => serde_json::json!({"raw": raw_stdout}).to_string(),
114    }
115}
116
117fn classify_run_outcome(
118    status: &str,
119    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
120    push_failed: bool,
121) -> &'static str {
122    if push_failed {
123        return "push_failed";
124    }
125    match parse_result {
126        Err(agents::AgentError::Timeout { .. }) => "timeout",
127        Err(agents::AgentError::Interrupted { .. }) => "interrupted",
128        // Generic rate-limit patterns (e.g. "usage limit") also match billing-cycle
129        // exhaustion messages, so consult the credit-exhaustion detector first —
130        // otherwise Kimi-style "usage limit for this billing cycle" errors are
131        // stored as a plain "rate_limit" outcome (issue #3529).
132        Err(agents::AgentError::RateLimit { message }) => {
133            cooldown::detect_credit_exhaustion_outcome(message).unwrap_or("rate_limit")
134        }
135        Err(agents::AgentError::Auth { message }) => {
136            cooldown::detect_credit_exhaustion_outcome(message).unwrap_or("failed")
137        }
138        Err(agents::AgentError::InvalidResponse { .. }) => "parse_error",
139        Ok(_)
140            if matches!(
141                status,
142                "done" | "completed" | "in_progress" | "in_review" | "needs_review" | "routed"
143            ) =>
144        {
145            "success"
146        }
147        Ok(_) if status == "blocked" => "blocked",
148        // Non-canonical status that clearly reads as completion — success, not
149        // "unrecognized status" (#3627).
150        Ok(_) if status_heuristics::status_looks_like_descriptive_completion(status) => "success",
151        Err(_) | Ok(_) => "failed",
152    }
153}
154
155fn classify_run_error_type(last_error: &str) -> &'static str {
156    if last_error.is_empty() {
157        // No error: agent completed successfully and created a PR waiting for review
158        "success"
159    } else if last_error.contains("timeout") {
160        "timeout"
161    } else if last_error.contains("interrupted") {
162        "interrupted"
163    } else if last_error.contains("billing cycle") || last_error.contains("monthly quota") {
164        "billing_cycle_exhausted"
165    } else if last_error.contains("rate limit") || last_error.contains("usage limit") {
166        "rate_limit"
167    } else if last_error.contains("auth") {
168        "auth_error"
169    } else if last_error.contains("push failed") {
170        "push_failed"
171    } else if last_error.contains("No commits between") || last_error.contains("no commits") {
172        "no_commits"
173    } else if last_error.contains("create PR failed")
174        || last_error.contains("failed to create pull request")
175        || last_error.contains("pull request creation failed")
176    {
177        "pr_failed"
178    } else if last_error.contains("invalid response") || last_error.contains("parse error") {
179        "parse_error"
180    } else if last_error.contains("exceeded max attempts") {
181        "max_attempts"
182    } else {
183        let lower = last_error.to_ascii_lowercase();
184        if lower.contains("build failed")
185            || lower.contains("tests failed")
186            || lower.contains("test failed")
187            || lower.contains("test suite failed")
188            || lower.contains("lint failed")
189        {
190            "ci_failure"
191        } else {
192            "failed"
193        }
194    }
195}
196
197fn extract_run_tokens(
198    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
199) -> (u64, u64) {
200    match parse_result {
201        Ok(parsed) => (
202            parsed
203                .input_tokens
204                .or(parsed.response.input_tokens)
205                .unwrap_or(0),
206            parsed
207                .output_tokens
208                .or(parsed.response.output_tokens)
209                .unwrap_or(0),
210        ),
211        Err(_) => (0, 0),
212    }
213}
214
215#[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
216fn run_duration_secs(started_at: &chrono::DateTime<Utc>) -> f64 {
217    (Utc::now() - *started_at).num_milliseconds() as f64 / 1000.0
218}
219
220struct RunAuditInput<'a> {
221    task_id: &'a str,
222    status: &'a str,
223    parse_result: &'a Result<agents::ParsedResponse, agents::AgentError>,
224    raw_stdout: &'a str,
225    raw_stderr: &'a str,
226    started_at: &'a chrono::DateTime<Utc>,
227    error_override: Option<String>,
228    elapsed_secs: Option<u64>,
229    push_failed: bool,
230}
231
232fn parse_success_output(
233    task_id: &str,
234    agent_name: &str,
235    agent_runner: &dyn agents::AgentRunner,
236    raw_stdout: &str,
237) -> Result<agents::ParsedResponse, agents::AgentError> {
238    // The last result envelope is the agent's final answer and carries the run's tokens and cost.
239    // Claude writes terminal_reason:completed on every run, so this goes before that shortcut
240    let agent_result = agents::find_agent_result(agent_name, raw_stdout);
241    if let Some(result) = &agent_result
242        && !result.is_error
243        && let Ok(response) = parser::parse(&result.result_text)
244    {
245        return Ok(agents::ParsedResponse {
246            response,
247            input_tokens: result.input_tokens,
248            output_tokens: result.output_tokens,
249            cost_usd: result.cost_usd,
250            duration_ms: result.duration_ms,
251        });
252    }
253
254    // Quick NDJSON terminal_reason:completed shortcut: when the agent
255    // emitted a session envelope that explicitly signals completion we
256    // prefer to interpret that as a successful run (or at least a
257    // soft/empty success) rather than immediately classifying an error
258    // from the trailing process output.  Do not override explicit
259    // in-envelope `is_error: true` signals.
260    if raw_stdout.contains("\"terminal_reason\":\"completed\"")
261        && !raw_stdout.contains("\"is_error\":true")
262    {
263        // Try to recover a structured response from assistant messages first
264        let assistant_text = agents::collect_assistant_messages_text(agent_name, raw_stdout);
265        if !assistant_text.is_empty() {
266            if let Ok(response) = parser::parse(&assistant_text) {
267                tracing::debug!(
268                    task_id,
269                    agent = agent_name,
270                    "found AgentResponse JSON in earlier assistant message (terminal_reason:completed)"
271                );
272                return Ok(agents::ParsedResponse {
273                    response,
274                    input_tokens: None,
275                    output_tokens: None,
276                    cost_usd: None,
277                    duration_ms: None,
278                });
279            }
280            if let Some(response) = agents::synthesize_response_from_text(&assistant_text) {
281                tracing::warn!(
282                    task_id,
283                    agent = agent_name,
284                    "synthesizing response from assistant message due to terminal_reason:completed"
285                );
286                return Ok(agents::ParsedResponse {
287                    response,
288                    input_tokens: None,
289                    output_tokens: None,
290                    cost_usd: None,
291                    duration_ms: None,
292                });
293            }
294        }
295        // If we couldn't recover structured data from earlier assistant
296        // messages, fall through and let the regular NDJSON envelope parsing
297        // / agent-specific parsers handle the terminal_reason case. This
298        // preserves the existing behavior where an empty `result` field
299        // combined with exit != 0 becomes an InvalidResponse that is then
300        // interpreted by parse_session_output (see tests).
301    }
302
303    // 1. Try to use the unified agent result extractor first
304    if let Some(agent_result) = agent_result {
305        // Treat is_error as authoritative for the success path
306        if agent_result.is_error {
307            // Classify the error from the agent's result text rather than
308            // blindly returning AgentFailed — this preserves rate-limit, auth,
309            // and other actionable error types for proper cooldown/reroute.
310            // Without this, e.g. a claude session-limit error with is_error=true
311            // would be downgraded to AgentFailed, losing the RateLimit signal
312            // that the fallback chain needs for UsageLimit recovery (issue #3292).
313            if let Some(e) = agents::patterns::detect_rate_limit(&agent_result.result_text) {
314                return Err(e);
315            }
316            if let Some(e) = agents::patterns::detect_auth_error(&agent_result.result_text) {
317                return Err(e);
318            }
319            if let Some(e) = agents::patterns::detect_model_unavailable(&agent_result.result_text) {
320                return Err(e);
321            }
322            return Err(agents::AgentError::AgentFailed {
323                message: agent_result.result_text,
324            });
325        }
326
327        // Strategy 2: scan all earlier assistant messages in the NDJSON
328        // stream for valid AgentResponse JSON. Agents often emit the
329        // JSON in an intermediate turn and then summarise in prose as
330        // their final output, so the `type:result` text may be pure
331        // narrative while a prior assistant turn contains the JSON.
332        let assistant_text = agents::collect_assistant_messages_text(agent_name, raw_stdout);
333        if !assistant_text.is_empty()
334            && let Ok(response) = parser::parse(&assistant_text)
335        {
336            tracing::debug!(
337                task_id,
338                agent = agent_name,
339                "found AgentResponse JSON in earlier assistant message"
340            );
341            return Ok(agents::ParsedResponse {
342                response,
343                input_tokens: agent_result.input_tokens,
344                output_tokens: agent_result.output_tokens,
345                cost_usd: agent_result.cost_usd,
346                duration_ms: agent_result.duration_ms,
347            });
348        }
349
350        // Only fall back to text synthesis when parsing fails
351        if let Some(response) = agents::synthesize_response_from_text(&agent_result.result_text) {
352            tracing::warn!(
353                task_id,
354                agent = agent_name,
355                "parse failed on agent result, synthesizing response from plain text"
356            );
357            return Ok(agents::ParsedResponse {
358                response,
359                input_tokens: agent_result.input_tokens,
360                output_tokens: agent_result.output_tokens,
361                cost_usd: agent_result.cost_usd,
362                duration_ms: agent_result.duration_ms,
363            });
364        }
365
366        // NDJSON envelope fallback: the envelope explicitly reports
367        // success (is_error=false) but the result content is domain
368        // JSON or plain prose that doesn't match AgentResponse schema
369        // and doesn't contain completion keywords.  Rather than
370        // returning InvalidResponse (which causes the task to re-run),
371        // synthesize a minimal "done" response from the result text.
372        //
373        // Guard against kimi/minimax edge case: when terminal_reason=completed
374        // but result text is empty, synthesize returns None, parse_success_output
375        // returns InvalidResponse, and parse_session_output can intercept it
376        // before classify_error. Requiring non-empty result here means the
377        // envelope fallback only fires when we have actual text to report.
378        if !agent_result.is_error
379            && result_envelope_completed(raw_stdout)
380            && !agent_result.result_text.is_empty()
381        {
382            let summary_end = agents::truncate_at_char_boundary(&agent_result.result_text, 500);
383            let summary = agent_result.result_text[..summary_end].to_string();
384            tracing::warn!(
385                task_id,
386                agent = agent_name,
387                "NDJSON envelope indicates success but result lacks AgentResponse \
388                 schema — synthesizing done response"
389            );
390            return Ok(agents::ParsedResponse {
391                response: AgentResponse {
392                    status: "done".to_string(),
393                    summary,
394                    ..Default::default()
395                },
396                input_tokens: agent_result.input_tokens,
397                output_tokens: agent_result.output_tokens,
398                cost_usd: agent_result.cost_usd,
399                duration_ms: agent_result.duration_ms,
400            });
401        }
402    }
403
404    // 2. Fall back to generic agent-specific parser if extraction failed
405    match agent_runner.parse_response(raw_stdout) {
406        Ok(parsed) => Ok(parsed),
407        Err(agents::AgentError::InvalidResponse { raw }) => {
408            if let Some(response) = agents::synthesize_response_from_text(&raw) {
409                tracing::warn!(
410                    task_id,
411                    agent = agent_name,
412                    "parse failed, synthesizing response from plain text"
413                );
414                // Try to extract token metadata from the original raw NDJSON
415                // output (not the truncated/extracted text from the error), so
416                // cost tracking isn't lost for agents that return free-form text.
417                let agent_result = agents::find_agent_result(agent_name, raw_stdout);
418                let input_tokens = agent_result.as_ref().and_then(|r| r.input_tokens);
419                let output_tokens = agent_result.as_ref().and_then(|r| r.output_tokens);
420                let cost_usd = agent_result.as_ref().and_then(|r| r.cost_usd);
421                let duration_ms = agent_result.as_ref().and_then(|r| r.duration_ms);
422                Ok(agents::ParsedResponse {
423                    response,
424                    input_tokens,
425                    output_tokens,
426                    cost_usd,
427                    duration_ms,
428                })
429            } else {
430                Err(agents::AgentError::InvalidResponse { raw })
431            }
432        }
433        Err(err) => Err(err),
434    }
435}
436
437/// Whether the last `type: result` line reports `subtype: success` or
438/// `terminal_reason: completed`. Grok omits `terminal_reason`.
439fn result_envelope_line(raw_stdout: &str) -> Option<serde_json::Value> {
440    raw_stdout
441        .lines()
442        .rev()
443        .filter_map(|l| serde_json::from_str::<serde_json::Value>(l.trim()).ok())
444        .find(|v| v.get("type").and_then(|t| t.as_str()) == Some("result"))
445}
446
447fn has_result_envelope(raw_stdout: &str) -> bool {
448    result_envelope_line(raw_stdout).is_some()
449}
450
451fn result_envelope_completed(raw_stdout: &str) -> bool {
452    result_envelope_line(raw_stdout).is_some_and(|v| {
453        v.get("subtype").and_then(|t| t.as_str()) == Some("success")
454            || v.get("terminal_reason").and_then(|t| t.as_str()) == Some("completed")
455    })
456}
457
458// A timeout kill has no result envelope. Salvage only the wrap-up shape: a successful
459// `git commit`, then only shell calls that all returned ok. Returns the last assistant text
460fn salvage_wrapup(raw: &str) -> Option<String> {
461    // (tool name, command, finished ok) in call order
462    let mut calls: Vec<(String, String, Option<bool>)> = Vec::new();
463    let mut last_text = String::new();
464    for v in raw
465        .lines()
466        .filter_map(|l| serde_json::from_str::<Value>(l.trim()).ok())
467    {
468        let blocks = v.pointer("/message/content").and_then(|c| c.as_array());
469        for b in blocks.into_iter().flatten() {
470            match b.get("type").and_then(|t| t.as_str()) {
471                Some("tool_use") => calls.push((
472                    b["name"].as_str().unwrap_or("").to_string(),
473                    b.pointer("/input/command")
474                        .and_then(|c| c.as_str())
475                        .unwrap_or("")
476                        .to_string(),
477                    None,
478                )),
479                Some("text") if v["type"] == "assistant" => {
480                    last_text = b["text"].as_str().unwrap_or("").to_string();
481                }
482                Some("tool_result") => {
483                    let ok = b.get("is_error").and_then(serde_json::Value::as_bool) != Some(true);
484                    // results arrive in call order, fill the first unanswered call
485                    if let Some(c) = calls.iter_mut().find(|c| c.2.is_none()) {
486                        c.2 = Some(ok);
487                    }
488                }
489                _ => {}
490            }
491        }
492    }
493    let commit = calls
494        .iter()
495        .rposition(|(_, cmd, ok)| cmd.contains("git commit") && *ok == Some(true))?;
496    let tail_ok = calls[commit..].iter().all(|(name, _, ok)| {
497        matches!(name.as_str(), "Bash" | "run_terminal_cmd") && *ok == Some(true)
498    });
499    let lower = last_text.to_lowercase();
500    if !tail_ok || lower.contains("still running") || lower.contains("in progress") {
501        return None;
502    }
503    Some(last_text.chars().take(500).collect())
504}
505
506/**
507 * Tries `parse_success_output` first when stdout is non-empty. If that succeeds,
508 * returns success regardless of exit code. If it fails and `exit_code != 0`,
509 * checks for `"terminal_reason":"completed"` or cost-telemetry-only stdout
510 * (`"costUSD":`) before falling back to `classify_error`, so valid sessions that
511 * exit 1 produce `InvalidResponse` (soft error, re-route) instead of a garbled
512 * `Unknown`/`Auth` error (hard failure with spurious cooldown)
513 **/
514pub fn parse_session_output(
515    task_id: &str,
516    agent_name: &str,
517    agent_runner: &dyn agents::AgentRunner,
518    session_output: &session::SessionOutput,
519) -> Result<agents::ParsedResponse, agents::AgentError> {
520    // Some agents may emit NDJSON/telemetry to stderr instead of stdout. Combine
521    // both streams so parse_success_output can inspect either place for the
522    // terminal_reason/cost telemetry envelope. Only skip parsing when both are
523    // empty so we still attempt recovery when stderr contains the envelope.
524    let combined_output = format!("{}{}", session_output.raw_stdout, session_output.raw_stderr);
525    if !(session_output.raw_stdout.is_empty() && session_output.raw_stderr.is_empty()) {
526        match parse_success_output(task_id, agent_name, agent_runner, &combined_output) {
527            Ok(parsed) => return Ok(parsed),
528            Err(err) => {
529                if session_output.exit_code == 124
530                    && let Some(summary) = salvage_wrapup(&combined_output)
531                {
532                    tracing::warn!(
533                        task_id,
534                        agent = agent_name,
535                        "timed out after commit during wrap-up, salvaging as done"
536                    );
537                    return Ok(agents::ParsedResponse {
538                        response: AgentResponse {
539                            status: "done".to_string(),
540                            summary,
541                            ..Default::default()
542                        },
543                        input_tokens: None,
544                        output_tokens: None,
545                        cost_usd: None,
546                        duration_ms: None,
547                    });
548                }
549                // If parse_success_output already returned a classified error
550                // (RateLimit, AgentFailed, Timeout, Auth, etc.), propagate it
551                // directly — only InvalidResponse (unparseable output) should
552                // be intercepted by the terminal_reason:completed / cost-telemetry
553                // guard below. Without this, e.g. a claude session-limit error
554                // with terminal_reason=completed would be downgraded from RateLimit
555                // to InvalidResponse → Failed, losing the correct cooldown/reroute.
556                if !matches!(err, agents::AgentError::InvalidResponse { .. }) {
557                    return Err(err);
558                }
559                if session_output.exit_code != 0 {
560                    // kimi/minimax/glm sometimes emit valid NDJSON with
561                    // "terminal_reason":"completed" or cost-only telemetry
562                    // ("costUSD":) but exit 1 due to a post-session hook or
563                    // minor CLI issue. Scanning the tail would just grab JSON
564                    // metadata, producing a garbled error. Detect either
565                    // completion signal and treat as a soft parse error so the
566                    // task re-routes instead of hitting a false cooldown from
567                    // Unknown or Auth error variants. Use the combined output
568                    // (stdout+stderr) so envelopes written to stderr are
569                    // observed.
570                    let combined = &combined_output;
571                    let is_completed_signal =
572                        combined.contains("\"terminal_reason\":\"completed\"");
573                    // GLM emits a cost-telemetry JSON object (containing
574                    // "costUSD":) as its final stdout/stderr line on clean
575                    // exit even when the process exits with code 1. If the
576                    // only content is such telemetry (no is_error=true in the
577                    // envelope), treat it the same as terminal_reason:completed.
578                    let is_cost_telemetry_only = combined.contains("\"costUSD\":")
579                        && !combined.contains("\"is_error\":true");
580                    if is_completed_signal || is_cost_telemetry_only {
581                        let raw_tail = agents::patterns::safe_tail(combined, 300);
582                        return Err(agents::AgentError::InvalidResponse {
583                            raw: raw_tail.to_string(),
584                        });
585                    }
586                    return Err(agent_runner.classify_error_with_elapsed(
587                        session_output.exit_code,
588                        &session_output.raw_stdout,
589                        &session_output.raw_stderr,
590                        session_output.elapsed_secs,
591                    ));
592                }
593                if !has_result_envelope(&combined_output) {
594                    return Err(agents::AgentError::Interrupted {
595                        message: "exited without a result envelope".to_string(),
596                    });
597                }
598                return Err(err);
599            }
600        }
601    }
602
603    if session_output.exit_code != 0 {
604        return Err(agent_runner.classify_error_with_elapsed(
605            session_output.exit_code,
606            &session_output.raw_stdout,
607            &session_output.raw_stderr,
608            session_output.elapsed_secs,
609        ));
610    }
611
612    Err(agents::AgentError::Unknown {
613        exit_code: session_output.exit_code,
614        message: "empty-output-exit0: opencode returned exit 0 with empty stdout".to_string(),
615    })
616}
617
618/// The repo has no project directory: not in the active config, no legacy
619/// `project_dir`, no bare clone. It fails before any agent starts
620#[derive(Debug)]
621pub(crate) struct NoProjectDir {
622    pub(crate) repo: String,
623    pub(crate) bare_clone: PathBuf,
624}
625
626impl NoProjectDir {
627    /// Block reason for a task of this repo
628    pub(crate) const BLOCK_REASON: &str = "project is not configured and has no bare clone";
629}
630
631impl fmt::Display for NoProjectDir {
632    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
633        write!(
634            f,
635            "no resolvable project directory for repo {} — not in active config, no bare clone at {}, no legacy project_dir set",
636            self.repo,
637            self.bare_clone.display()
638        )
639    }
640}
641
642impl error::Error for NoProjectDir {}
643
644/// Task runner configuration.
645pub struct TaskRunner {
646    /// Repository slug (owner/repo)
647    repo: String,
648    /// Path to the orch home directory
649    orch_home: PathBuf,
650    /// Unified task store for run audit trail
651    store: Option<Arc<TaskStore>>,
652}
653
654impl Debug for TaskRunner {
655    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
656        f.debug_struct("TaskRunner")
657            .field("repo", &self.repo)
658            .field("orch_home", &self.orch_home)
659            .finish_non_exhaustive()
660    }
661}
662
663impl TaskRunner {
664    pub fn new(repo: String) -> Self {
665        let orch_home = home::orch_home().unwrap_or_else(|e| {
666            tracing::warn!(error = %e, "orch_home() failed — falling back to /tmp/.orch");
667            PathBuf::from("/tmp").join(".orch")
668        });
669
670        Self {
671            repo,
672            orch_home,
673            store: None,
674        }
675    }
676
677    /// Set the unified task store for run audit trail.
678    #[must_use]
679    pub fn with_store(mut self, store: Arc<TaskStore>) -> Self {
680        self.store = Some(store);
681        self
682    }
683
684    /// Read a field from the task store.
685    async fn get_field(&self, task_id: &str, field: &str) -> Option<String> {
686        let task = store::opt_store_get_task(&self.store, &self.repo, task_id).await?;
687        match field {
688            "route_reason" => Some(task.route_reason),
689            "worktree" => Some(task.worktree),
690            "attempts" => Some(task.attempts.to_string()),
691            "delegations" => serde_json::to_string(&task.delegations).ok(),
692            "summary" => Some(task.summary),
693            "last_error" => Some(task.last_error),
694            "agent" => task.agent,
695            _ => None,
696        }
697    }
698
699    /// Keep a stored fallback on reroute, wipe agent/model when the run's agent is still stored
700    pub(crate) async fn clear_agent_on_reroute_unless_fallback(
701        &self,
702        task_id: &str,
703        ran_agent: &str,
704    ) {
705        let stored = store::opt_store_get_task(&self.store, &self.repo, task_id).await;
706        if Self::reroute_keeps_stored_agent(stored.as_ref(), ran_agent) {
707            tracing::debug!(
708                task_id,
709                "stored fallback agent present, skipping agent/model clear"
710            );
711            return;
712        }
713        if let Err(e) = store::store_set_result(
714            &self.store,
715            &self.repo,
716            task_id,
717            &[
718                ("agent", serde_json::json!("")),
719                ("model", serde_json::json!("")),
720            ],
721        )
722        .await
723        {
724            tracing::warn!(task_id, err = %e, "failed to clear agent/model for reroute in store");
725        }
726    }
727
728    /// True when a fallback is already stored and must survive this reroute
729    fn reroute_keeps_stored_agent(stored: Option<&store::Task>, ran_agent: &str) -> bool {
730        let Some(task) = stored else {
731            return false;
732        };
733        if !matches!(task.status, TaskStatus::InProgress) {
734            return true;
735        }
736        let stored_agent = task.agent.as_deref().unwrap_or("");
737        !stored_agent.is_empty() && stored_agent != ran_agent
738    }
739
740    async fn build_run_audit(&self, input: RunAuditInput<'_>) -> RunAudit {
741        // Filter empty strings so that a blank last_error does not shadow the
742        // actual parse_result error.  Without this, `Option::or(Some(""))` would
743        // return `Some("")` and prevent the `unwrap_or_else` fallback from ever
744        // running, causing task_runs.error to be written as "" even when the
745        // agent returned a classifiable error (fixes issue #2527).
746        let last_error = self
747            .get_field(input.task_id, "last_error")
748            .await
749            .filter(|s| !s.is_empty());
750        let outcome =
751            classify_run_outcome(input.status, input.parse_result, input.push_failed).to_string();
752
753        let error = if outcome == "success" {
754            String::new()
755        } else {
756            // Prefer this run's parsed error over a last_error left by a previous
757            // agent's failover. last_error stays the Err/EarlyReturn fallback (#3851)
758            let error = input
759                .error_override
760                .as_deref()
761                .filter(|s| !s.is_empty())
762                .map(str::to_string)
763                .or_else(|| match input.parse_result {
764                    Ok(parsed) => parsed
765                        .response
766                        .error
767                        .as_deref()
768                        .filter(|s| !s.is_empty())
769                        .map(str::to_string),
770                    Err(_) => None,
771                })
772                .or(last_error)
773                .unwrap_or_else(|| match input.parse_result {
774                    Ok(_) => String::new(),
775                    Err(err) => err.to_string(),
776                });
777
778            if error.is_empty() {
779                if input.status == "blocked" {
780                    match input.parse_result {
781                        Ok(parsed) if !parsed.response.summary.is_empty() => {
782                            parsed.response.summary.clone()
783                        }
784                        _ => "task blocked without explicit error".to_string(),
785                    }
786                } else if matches!(
787                    input.status,
788                    "done"
789                        | "completed"
790                        | "in_progress"
791                        | "in_review"
792                        | "needs_review"
793                        | "routed"
794                        | "new"
795                ) {
796                    // Canonical non-success statuses have no error message (e.g. "routed").
797                    String::new()
798                } else {
799                    // Non-canonical status with no error message — surface the status
800                    // explicitly so operators can identify which statuses need to be
801                    // added to the normalization map in parser.rs.
802                    format!("unrecognized status: {}", input.status)
803                }
804            } else {
805                error
806            }
807        };
808
809        let total_cost_usd = input
810            .parse_result
811            .as_ref()
812            .ok()
813            .and_then(|p| p.cost_usd)
814            .unwrap_or(0.0);
815
816        let (input_tokens, output_tokens) = extract_run_tokens(input.parse_result);
817
818        #[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
819        let duration_secs = input
820            .elapsed_secs
821            .map_or_else(|| run_duration_secs(input.started_at), |s| s as f64);
822
823        RunAudit {
824            stdout: input.raw_stdout.to_string(),
825            stderr: input.raw_stderr.to_string(),
826            parsed_response: serialize_parsed_response(input.parse_result, input.raw_stdout),
827            outcome,
828            error,
829            input_tokens,
830            output_tokens,
831            total_cost_usd,
832            duration_secs,
833        }
834    }
835
836    /// The blocked outcome for a sandbox that cannot start. No agent ran, so none is cooled down
837    async fn sandbox_blocked(&self, task_id: &str, reason: String) -> RunExecution {
838        tracing::error!(task_id, error = %reason, "blocking task");
839        if let Err(e) = store::store_set_result(
840            &self.store,
841            &self.repo,
842            task_id,
843            &[
844                ("last_error", serde_json::json!(reason)),
845                ("block_reason", serde_json::json!(reason)),
846            ],
847        )
848        .await
849        {
850            tracing::warn!(task_id, err = %e, "failed to write the sandbox block_reason");
851        }
852        RunExecution {
853            status: "blocked".to_string(),
854            exit_code: None,
855            audit: RunAudit {
856                stdout: String::new(),
857                stderr: String::new(),
858                parsed_response: String::new(),
859                outcome: "failed".to_string(),
860                error: reason,
861                input_tokens: 0,
862                output_tokens: 0,
863                total_cost_usd: 0.0,
864                duration_secs: 0.0,
865            },
866        }
867    }
868
869    /// Run a task through the full execution pipeline.
870    ///
871    /// Returns `Ok(None)` if the runner guard skipped the task (caller should not
872    /// post stale results). Returns `Ok(Some(status))` with the final task status.
873    pub async fn run(
874        &self,
875        task_id: &str,
876        agent: Option<&str>,
877        model: Option<&str>,
878        backend: Option<&dyn ExternalBackend>,
879        started_at: &chrono::DateTime<Utc>,
880    ) -> anyhow::Result<Option<RunExecution>> {
881        tracing::info!(
882            task_id,
883            agent = agent.unwrap_or("default"),
884            model = model.unwrap_or("default"),
885            "starting task execution"
886        );
887
888        // Check task guards; returns outcome indicating whether to proceed.
889        let attempts = match task_init::check_guards(task_id, &self.repo, &self.store).await {
890            Ok(task_init::GuardOutcome::Proceed(a)) => a,
891            Ok(task_init::GuardOutcome::Skip) => {
892                return Ok(None);
893            }
894            Ok(task_init::GuardOutcome::MaxAttempts) => {
895                // The caller marks the task blocked from the returned status, a task that ran out
896                // of attempts did not finish, so it must never reach done
897                if backend.is_some() {
898                    // If this was label-forced to a specific agent, remove the agent label
899                    // so that /retry can route to a different agent instead of looping.
900                    let route_reason = self
901                        .get_field(task_id, "route_reason")
902                        .await
903                        .unwrap_or_default();
904                    if route_reason.starts_with("label agent:") {
905                        let agent_label = route_reason.trim_start_matches("label ");
906                        let gh = GhHttp::new()?;
907                        match gh.remove_label(&self.repo, task_id, agent_label).await {
908                            Err(e) => {
909                                tracing::warn!(task_id, label = agent_label, err = %e, "failed to remove agent label after max attempts");
910                            }
911                            _ => {
912                                tracing::info!(
913                                    task_id,
914                                    label = agent_label,
915                                    "removed forced agent label after max attempts — /retry will use free routing"
916                                );
917                            }
918                        }
919                    }
920                }
921                return Ok(Some(RunExecution {
922                    status: "blocked".to_string(),
923                    exit_code: None,
924                    audit: RunAudit {
925                        stdout: String::new(),
926                        stderr: String::new(),
927                        parsed_response: String::new(),
928                        outcome: "failed".to_string(),
929                        error: "max attempts reached".to_string(),
930                        input_tokens: 0,
931                        output_tokens: 0,
932                        total_cost_usd: 0.0,
933                        duration_secs: 0.0,
934                    },
935                }));
936            }
937            Err(e) => return Err(e),
938        };
939
940        // Resolve project directory
941        let project_dir = self.resolve_project_dir().await?;
942
943        // Prepare task: worktree, context, prompts, and agent invocation
944        let init = task_init::prepare_task(
945            task_id,
946            agent,
947            model,
948            backend,
949            &self.repo,
950            &project_dir,
951            attempts,
952            &self.store,
953        )
954        .await?;
955
956        // A sandbox that cannot start blocks the task, which never runs without it. It is the
957        // operator's setup that fails, so no agent or model is cooled down
958        if init.invocation.sandbox
959            && let Err(e) = sandbox::argv(
960                &init.invocation,
961                &env::temp_dir().join("orch-sandbox-preflight"),
962            )
963            .await
964        {
965            return Ok(Some(self.sandbox_blocked(task_id, e.to_string()).await));
966        }
967
968        // Run agent session: spawn in tmux, wait for completion, collect output
969        let (tmux, tmux_session, session_output) = session::run_agent_session(
970            task_id,
971            &init.invocation,
972            &init.attempt_dir,
973            &self.orch_home,
974        )
975        .await;
976
977        // The pre-flight policy passed but the one built at spawn did not: block the same way
978        if let Some(reason) = session_output.sandbox_error.clone() {
979            return Ok(Some(self.sandbox_blocked(task_id, reason).await));
980        }
981
982        // Touch updated_at immediately after session exit so the stuck-task recovery
983        // timer resets before response_handler post-processing (git push, PR creation,
984        // status update) begins.  Without this, a session that ran longer than
985        // no_session_stuck_timeout (default 10 min) would be re-dispatched by the tick
986        // while the runner is still writing results.
987        store::store_touch_updated_at(&self.store, &self.repo, task_id).await;
988
989        // If silence detection (tick phase1b) already rerouted this task while the
990        // session was running, skip all fallback/review processing. Without this check the
991        // runner would call `fallback::handle_error` on the empty output, exhaust fallback
992        // agents, and mark the task `needs_review` — overwriting the rerouted status and
993        // triggering a spurious review cycle.
994        // Silence detection now sets Routed (with fallback agent) or NeedsReview (no fallback).
995        // Check for both, plus New for backward compatibility.
996        if let Some(stored) = store::opt_store_get_task(&self.store, &self.repo, task_id).await {
997            let silence_reset = matches!(
998                stored.status,
999                TaskStatus::New | TaskStatus::Routed | TaskStatus::NeedsReview
1000            );
1001            if silence_reset {
1002                let status_str = stored.status.as_str();
1003                tracing::info!(
1004                    task_id,
1005                    status = status_str,
1006                    "task already reset by the engine — skipping fallback processing"
1007                );
1008                session::cleanup_session(task_id, &tmux, &tmux_session).await;
1009                let silence_err = agents::patterns::classify_from_text(0, "silence detected");
1010                let silence_parse: Result<agents::ParsedResponse, agents::AgentError> =
1011                    Err(silence_err);
1012                let audit = self
1013                    .build_run_audit(RunAuditInput {
1014                        task_id,
1015                        status: status_str,
1016                        parse_result: &silence_parse,
1017                        raw_stdout: &session_output.raw_stdout,
1018                        raw_stderr: &session_output.raw_stderr,
1019                        started_at,
1020                        error_override: Some(format!("task was reset by the engine while the session ran (status {status_str})")),
1021                        elapsed_secs: session_output.elapsed_secs,
1022                        push_failed: false,
1023                    })
1024                    .await;
1025                return Ok(Some(RunExecution {
1026                    status: status_str.to_string(),
1027                    exit_code: Some(session_output.exit_code),
1028                    audit,
1029                }));
1030            }
1031        }
1032
1033        // Log raw output for debugging agent failures
1034        let stdout_len = session_output.raw_stdout.len();
1035        let stderr_len = session_output.raw_stderr.len();
1036        let stdout_tail: String = session_output
1037            .raw_stdout
1038            .chars()
1039            .rev()
1040            .take(500)
1041            .collect::<String>()
1042            .chars()
1043            .rev()
1044            .collect();
1045        let stderr_tail: String = session_output
1046            .raw_stderr
1047            .chars()
1048            .rev()
1049            .take(500)
1050            .collect::<String>()
1051            .chars()
1052            .rev()
1053            .collect();
1054        tracing::info!(
1055            task_id,
1056            exit_code = session_output.exit_code,
1057            stdout_len,
1058            stderr_len,
1059            stdout_tail = %stdout_tail,
1060            stderr_tail = %stderr_tail,
1061            "agent raw output"
1062        );
1063
1064        // Use agent-specific parsing where possible. Previously we only attempted
1065        // parse_success_output when exit_code == 0 which caused runs that emitted
1066        // valid NDJSON but exited non-zero (kimi / minimax) to be classed as
1067        // failures. Try parsing success output first when stdout is present; if
1068        // parsing succeeds treat as success regardless of exit code. Only fall
1069        // back to error classification when parsing fails and the process exit
1070        // code indicates an error.
1071        for window in agents::claude::rate_limit_windows(&session_output.raw_stdout) {
1072            cooldown::record_rate_limit_window(&init.agent_name, &window).await;
1073        }
1074        let agent_runner = agents::get_runner(&init.agent_name);
1075        let parse_result =
1076            parse_session_output(task_id, &init.agent_name, &*agent_runner, &session_output);
1077
1078        // Write structured result.json for deterministic testing and debugging
1079        response_handler::write_result_json(
1080            &init.attempt_dir,
1081            task_id,
1082            &init.agent_name,
1083            init.model_name.as_deref(),
1084            session_output.exit_code,
1085            init.new_attempts,
1086            &parse_result,
1087            &session_output.raw_stdout,
1088            &session_output.raw_stderr,
1089        )
1090        .await;
1091
1092        // Handle outcome: success or error recovery
1093        // `fallback_error` carries the attributed error message from `handle_error()` when
1094        // the Continue path is taken.  It is passed as `error_override` to `build_run_audit`
1095        // so that stale `last_error` values from a previous agent's run are never written
1096        // into the current run's audit record (fixes misattribution bug: task_run shows
1097        // agent=kimi but error="minimax timed out...").
1098        let (final_status, push_failed, fallback_error) = match parse_result {
1099            Ok(ref parsed) => {
1100                let (status, push_failed) = response_handler::handle_success(
1101                    task_id,
1102                    parsed.clone(),
1103                    &init.wt,
1104                    &init.task_title,
1105                    &init.agent_name,
1106                    init.model_name.as_deref(),
1107                    init.new_attempts,
1108                    &self.repo,
1109                    &self.store,
1110                    &session_output.raw_stdout,
1111                )
1112                .await?;
1113                (status, push_failed, None)
1114            }
1115            Err(ref agent_err) => {
1116                match fallback::handle_error(
1117                    task_id,
1118                    agent_err,
1119                    &init.agent_name,
1120                    &*agent_runner,
1121                    init.model_name.as_deref(),
1122                    init.complexity.as_deref(),
1123                    init.new_attempts,
1124                    &self.store,
1125                    &self.repo,
1126                )
1127                .await?
1128                {
1129                    fallback::ErrorHandleResult::EarlyReturn { status } => {
1130                        // Task rerouted — clean up tmux session and env secrets
1131                        session::cleanup_session(task_id, &tmux, &tmux_session).await;
1132                        // The fallback handler already writes a sanitized `last_error`
1133                        // into the store for EarlyReturn paths (reroutes / needs_review).
1134                        // Passing the raw agent_err here would record noisy provider
1135                        // payloads (e.g. full api_retry NDJSON). Leave error_override
1136                        // as None so build_run_audit will prefer the store's last_error
1137                        // (see src/engine/runner/fallback.rs which persists a compact
1138                        // summary for rate-limit and reroute cases).
1139                        let audit = self
1140                            .build_run_audit(RunAuditInput {
1141                                task_id,
1142                                status: &status,
1143                                parse_result: &parse_result,
1144                                raw_stdout: &session_output.raw_stdout,
1145                                raw_stderr: &session_output.raw_stderr,
1146                                started_at,
1147                                error_override: None,
1148                                elapsed_secs: session_output.elapsed_secs,
1149                                push_failed: false,
1150                            })
1151                            .await;
1152                        return Ok(Some(RunExecution {
1153                            status,
1154                            exit_code: Some(session_output.exit_code),
1155                            audit,
1156                        }));
1157                    }
1158                    fallback::ErrorHandleResult::Continue { status, error } => {
1159                        (status, false, Some(error))
1160                    }
1161                }
1162            }
1163        };
1164
1165        // Kill tmux session if still alive
1166        session::cleanup_session(task_id, &tmux, &tmux_session).await;
1167
1168        // For no-op done tasks (agent finished with no code changes and no PR),
1169        // clean up the worktree and branch immediately — tmux session is already
1170        // dead so the tmux guard won't block removal.
1171        if final_status.as_str() == "done"
1172            && let Some(ref st) = self.store
1173            && let Err(e) = cleanup::cleanup_task_worktree(task_id, &self.repo, st).await
1174        {
1175            tracing::warn!(task_id, error = ?e, "worktree cleanup failed after no-op done");
1176        }
1177
1178        // Return final status and the session exit code so callers can record it.
1179        // Use `fallback_error` as error_override when available: it carries the error message
1180        // attributed to the current agent (set by handle_error), preventing stale last_error
1181        // values from a prior run from contaminating this run's audit record.
1182        let audit = self
1183            .build_run_audit(RunAuditInput {
1184                task_id,
1185                status: &final_status,
1186                parse_result: &parse_result,
1187                raw_stdout: &session_output.raw_stdout,
1188                raw_stderr: &session_output.raw_stderr,
1189                started_at,
1190                error_override: fallback_error,
1191                elapsed_secs: session_output.elapsed_secs,
1192                push_failed,
1193            })
1194            .await;
1195
1196        Ok(Some(RunExecution {
1197            status: final_status.clone(),
1198            exit_code: Some(session_output.exit_code),
1199            audit,
1200        }))
1201    }
1202
1203    /// Record task execution metrics to the database.
1204    #[expect(
1205        clippy::too_many_arguments,
1206        reason = "a metric row needs the agent, model, route, timing and outcome together"
1207    )]
1208    async fn record_metrics(
1209        &self,
1210        task_id: &str,
1211        agent_name: &str,
1212        model_name: Option<&str>,
1213        route_result: Option<&RouteResult>,
1214        started_at: &chrono::DateTime<Utc>,
1215        attempts: u32,
1216        final_status: &str,
1217        error_type: Option<&str>,
1218    ) {
1219        let completed_at = Utc::now();
1220        #[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
1221        let duration_seconds = (completed_at - *started_at).num_milliseconds() as f64 / 1000.0;
1222
1223        // Classify outcome based on the final status and optional error_type/last_error
1224        let outcome = match final_status {
1225            "done" | "in_progress" | "in_review" => "success",
1226            "new" => "rerouted",
1227            "needs_review" => classify_run_error_type(error_type.unwrap_or("")),
1228            _ => "unknown",
1229        };
1230
1231        let complexity = route_result.map(|r| r.complexity.clone());
1232        let files_changed = git_ops::count_changed_files(&PathBuf::from(
1233            self.get_field(task_id, "worktree")
1234                .await
1235                .unwrap_or_default(),
1236        ))
1237        .await
1238        .unwrap_or(0);
1239
1240        if let Some(ref store) = self.store {
1241            // Only set error_type for non-success outcomes
1242            let db_error_type: Option<String> = if outcome == "success" {
1243                None
1244            } else {
1245                Some(outcome.to_string())
1246            };
1247
1248            let usage = store::get_token_usage(&self.store, &self.repo, task_id).await;
1249            let total_cost = store::get_reported_cost(&self.store, &self.repo, task_id).await;
1250            let input_tokens = if usage.input_tokens > 0 {
1251                Some(i64::try_from(usage.input_tokens).unwrap_or(i64::MAX))
1252            } else {
1253                None
1254            };
1255            let output_tokens = if usage.output_tokens > 0 {
1256                Some(i64::try_from(usage.output_tokens).unwrap_or(i64::MAX))
1257            } else {
1258                None
1259            };
1260
1261            let metric = InsertTaskMetric {
1262                repo: &self.repo,
1263                task_id,
1264                agent: agent_name,
1265                model: model_name,
1266                complexity: complexity.as_deref(),
1267                outcome,
1268                duration_seconds,
1269                started_at,
1270                completed_at: &completed_at,
1271                attempts: i32::try_from(attempts.saturating_add(1)).unwrap_or(i32::MAX),
1272                files_changed: i32::try_from(files_changed).unwrap_or(i32::MAX),
1273                error_type: db_error_type.as_deref(),
1274                input_tokens,
1275                output_tokens,
1276                total_cost_usd: total_cost,
1277            };
1278
1279            if let Err(e) = store.insert_task_metric(&metric).await {
1280                tracing::error!(task_id, ?e, "failed to record task metrics");
1281            }
1282        }
1283    }
1284
1285    /// Run a task with full engine context (backend, tmux, capture).
1286    ///
1287    /// Called by the engine dispatch loop with richer context.
1288    /// Returns a `WeightSignal` for the engine to feed back to the router.
1289    pub async fn run_with_context(
1290        &self,
1291        task: &ExternalTask,
1292        backend: &Arc<dyn ExternalBackend>,
1293        _tmux: &Arc<TmuxManager>,
1294        route_result: Option<&RouteResult>,
1295    ) -> anyhow::Result<WeightSignal> {
1296        let task_id = &task.id.0;
1297        let agent = route_result.map(|r| r.agent.as_str());
1298        let agent_name = agent.unwrap_or("claude").to_string();
1299        let model = route_result.and_then(|r| r.model.as_deref());
1300
1301        // Record start time for metrics (before any work begins)
1302        let started_at = Utc::now();
1303
1304        // Store task info for prompt building (title/body already in store tasks table)
1305
1306        // Record run start in task_runs audit trail
1307        let run_audit_id = if let Some(ref store) = self.store {
1308            let attempt: i32 = self
1309                .get_field(task_id, "attempts")
1310                .await
1311                .and_then(|s| s.parse().ok())
1312                .unwrap_or(0)
1313                + 1;
1314            match store.resolve_task_id(&self.repo, task_id).await {
1315                Ok(Some(store_id)) => {
1316                    match store
1317                        .start_run(&crate::store::StartRun {
1318                            task_id: store_id,
1319                            attempt,
1320                            run_type: "agent",
1321                            agent: &agent_name,
1322                            model: model.unwrap_or(""),
1323                            command: &format!(
1324                                "{} --model {}",
1325                                agent_name,
1326                                model.unwrap_or("default")
1327                            ),
1328                            prompt: &task.body,
1329                        })
1330                        .await
1331                    {
1332                        Ok(run_id) => Some(run_id),
1333                        Err(e) => {
1334                            tracing::warn!(
1335                                task_id,
1336                                error = %e,
1337                                "failed to record run start in audit trail"
1338                            );
1339                            None
1340                        }
1341                    }
1342                }
1343                _ => None,
1344            }
1345        } else {
1346            None
1347        };
1348
1349        // Run the task.
1350        // Always finalize the run audit row, even on early-return/error paths.
1351        let run_result = match self
1352            .run(task_id, agent, model, Some(&**backend), &started_at)
1353            .await
1354        {
1355            Ok(run_result) => run_result,
1356            Err(e) => {
1357                if let Some(run_id) = run_audit_id
1358                    && let Some(ref store) = self.store
1359                    && let Err(complete_err) = store
1360                        .complete_run(&crate::store::CompleteRun {
1361                            run_id,
1362                            exit_code: Some(-1),
1363                            stdout: "",
1364                            stderr: "",
1365                            parsed: "",
1366                            outcome: "failed",
1367                            error: &e.to_string(),
1368                            tokens: RunTokenUsage::default(),
1369                        })
1370                        .await
1371                {
1372                    tracing::warn!(
1373                        task_id,
1374                        run_id,
1375                        error = %complete_err,
1376                        "failed to record run completion in audit trail after runner error"
1377                    );
1378                }
1379                return Err(e);
1380            }
1381        };
1382
1383        // If the runner guard skipped the task, do not re-post stale data as a new comment.
1384        let (status, exit_code_opt, run_audit) = if let Some(result) = run_result {
1385            (result.status, result.exit_code, result.audit)
1386        } else {
1387            if let Some(run_id) = run_audit_id
1388                && let Some(ref store) = self.store
1389                && let Err(e) = store
1390                    .complete_run(&crate::store::CompleteRun {
1391                        run_id,
1392                        exit_code: Some(-1),
1393                        stdout: "",
1394                        stderr: "",
1395                        parsed: "",
1396                        outcome: "aborted",
1397                        error: "run skipped by guard",
1398                        tokens: RunTokenUsage::default(),
1399                    })
1400                    .await
1401            {
1402                tracing::warn!(
1403                    task_id,
1404                    run_id,
1405                    error = %e,
1406                    "failed to record skipped run completion in audit trail"
1407                );
1408            }
1409            tracing::info!(task_id, "guard skipped task — not posting stale result");
1410            return Ok(WeightSignal::None);
1411        };
1412
1413        // Process delegations if the agent requested subtasks
1414        let delegations_raw = self
1415            .get_field(task_id, "delegations")
1416            .await
1417            .unwrap_or_default();
1418        if !delegations_raw.is_empty() {
1419            match serde_json::from_str::<Vec<Delegation>>(&delegations_raw) {
1420                Ok(delegations) if !delegations.is_empty() => {
1421                    self.process_delegations(task, &delegations, backend)
1422                        .await?;
1423                    if let Err(e) = store::store_set_result(
1424                        &self.store,
1425                        &self.repo,
1426                        task_id,
1427                        &[("delegations", serde_json::json!([]))],
1428                    )
1429                    .await
1430                    {
1431                        tracing::warn!(task_id, err = %e, "failed to clear delegations in store");
1432                    }
1433                }
1434                Ok(_) => {} // empty list, nothing to do
1435                Err(e) => {
1436                    tracing::error!(task_id, error = %e, "corrupt delegations JSON — clearing");
1437                    if let Err(e2) = store::store_set_result(
1438                        &self.store,
1439                        &self.repo,
1440                        task_id,
1441                        &[("delegations", serde_json::json!([]))],
1442                    )
1443                    .await
1444                    {
1445                        tracing::warn!(task_id, err = %e2, "failed to clear corrupt delegations in store");
1446                    }
1447                    if let Err(e2) = store::store_set_result(
1448                        &self.store,
1449                        &self.repo,
1450                        task_id,
1451                        &[(
1452                            "last_error",
1453                            serde_json::json!(format!("delegation parse failed: {e}")),
1454                        )],
1455                    )
1456                    .await
1457                    {
1458                        tracing::warn!(task_id, err = %e2, "failed to write delegation_parse_failed last_error to store");
1459                    }
1460                }
1461            }
1462        }
1463
1464        // Post result to GitHub
1465        let summary = self.get_field(task_id, "summary").await.unwrap_or_default();
1466        let last_error = self
1467            .get_field(task_id, "last_error")
1468            .await
1469            .unwrap_or_default();
1470
1471        // Determine weight signal based on outcome.
1472        // Any reroute ("new"/"routed") should avoid review-gate and go back through
1473        // normal dispatch, regardless of why it was rerouted.
1474        let is_rerouted = status == "new" || status == "routed";
1475        if is_rerouted {
1476            self.clear_agent_on_reroute_unless_fallback(task_id, &agent_name)
1477                .await;
1478        }
1479        // Determine weight signal based on outcome.
1480        // Reroutes should only emit RateLimited when the underlying error is genuinely
1481        // a rate limit (not silence detection, timeouts, parse errors, etc.). This prevents
1482        // false cooldown cascades where non-rate-limit failures are treated as rate limits.
1483        let is_rate_limit_error = classify_run_error_type(&last_error) == "rate_limit";
1484        let weight_signal = if is_rerouted {
1485            if is_rate_limit_error {
1486                WeightSignal::RateLimited {
1487                    agent: agent_name.clone(),
1488                }
1489            } else {
1490                // Non-rate-limit reroutes (silence detection, timeout, parse error): signal
1491                // the engine to keep the task in "new" for re-routing. Using Rerouted (not
1492                // None) prevents tick from converting the silence-reset back to needs_review,
1493                // which would trigger the no-code review shortcut and falsely mark it done.
1494                WeightSignal::Rerouted
1495            }
1496        } else if status == "needs_review" {
1497            // Rate limit errors that escalate to needs_review must NOT reset backoff
1498            // counters (issue #2153).  These errors indicate the agent is genuinely
1499            // unavailable — resetting would wipe the cooldown and cause the next dispatch
1500            // to retry the same rate-limited agent immediately.
1501            //
1502            // Detection: classify_run_error_type() confirms "rate limit" in last_error.
1503            // Reuse the same classifier as `is_rate_limit_error` above rather than a raw
1504            // substring check — billing-cycle exhaustion messages (e.g. Kimi's "usage
1505            // limit for this billing cycle") also contain "usage limit"/"rate limit", so
1506            // a bare substring check would record a rate-limit event into the router's
1507            // health signal for a condition that isn't a genuine rate limit (issue #3529).
1508            let signal = needs_review_signal(&last_error, &agent_name);
1509            match &signal {
1510                WeightSignal::RateLimited { .. } => {
1511                    // Record in the metrics store so the router's health check detects it.
1512                    if let Some(ref store) = self.store
1513                        && let Err(e) = store
1514                            .record_rate_limit(&agent_name, "rate", Some(task_id))
1515                            .await
1516                    {
1517                        tracing::warn!(task_id, agent = %agent_name, err = %e, "failed to persist rate_limit event — router health check may miss this signal");
1518                    }
1519                    // Also record via the cooldown system so model_for_complexity() skips
1520                    // this model.
1521                    cooldown::record_rate_limit(&agent_name, model, &last_error).await;
1522                }
1523                WeightSignal::Success { .. } => {
1524                    // Reset backoff so the next failure starts fresh.
1525                    cooldown::record_agent_success(&agent_name, model.unwrap_or("")).await;
1526                }
1527                _ => {}
1528            }
1529            signal
1530        } else if status == "done" || status == "in_progress" || status == "in_review" {
1531            // Reset exponential backoff counters so the next failure starts fresh.
1532            cooldown::record_agent_success(&agent_name, model.unwrap_or("")).await;
1533            WeightSignal::Success {
1534                agent: agent_name.clone(),
1535            }
1536        } else if status == "blocked" {
1537            WeightSignal::Blocked
1538        } else {
1539            WeightSignal::None
1540        };
1541
1542        // Record metrics
1543        {
1544            let attempts: u32 = self
1545                .get_field(task_id, "attempts")
1546                .await
1547                .and_then(|s| s.parse().ok())
1548                .unwrap_or(0);
1549            self.record_metrics(
1550                task_id,
1551                &agent_name,
1552                model,
1553                route_result,
1554                &started_at,
1555                attempts,
1556                &status,
1557                if last_error.is_empty() {
1558                    None
1559                } else {
1560                    Some(&last_error)
1561                },
1562            )
1563            .await;
1564        }
1565
1566        // Complete run in task_runs audit trail (include exit code from runner when available)
1567        if let Some(run_id) = run_audit_id
1568            && let Some(ref store) = self.store
1569            && let Err(e) = store
1570                .complete_run(&crate::store::CompleteRun {
1571                    run_id,
1572                    exit_code: exit_code_opt,
1573                    stdout: &run_audit.stdout,
1574                    stderr: &run_audit.stderr,
1575                    parsed: &run_audit.parsed_response,
1576                    outcome: &run_audit.outcome,
1577                    error: &run_audit.error,
1578                    tokens: RunTokenUsage {
1579                        input_tokens: run_audit.input_tokens,
1580                        output_tokens: run_audit.output_tokens,
1581                        total_cost_usd: run_audit.total_cost_usd,
1582                        duration_secs: run_audit.duration_secs,
1583                    },
1584                })
1585                .await
1586        {
1587            tracing::warn!(
1588                task_id,
1589                run_id,
1590                error = %e,
1591                "failed to record run completion in audit trail"
1592            );
1593        }
1594
1595        // If task was rerouted (status=routed or new after run), update GitHub agent label
1596        // so the router doesn't re-route back to the same failed agent.
1597        if is_rerouted && !is_internal_id(task_id) {
1598            let new_agent = self.get_field(task_id, "agent").await.unwrap_or_default();
1599            if !new_agent.is_empty() && new_agent != agent_name {
1600                update_agent_label_after_reroute(
1601                    &self.repo,
1602                    task,
1603                    backend,
1604                    &agent_name,
1605                    &new_agent,
1606                )
1607                .await;
1608            }
1609        }
1610
1611        // Internal tasks: skip GitHub status/comment updates — dispatch phase handles status.
1612        if is_internal_id(task_id) {
1613            return Ok(weight_signal);
1614        }
1615
1616        // Update GitHub status
1617        let new_status = match status.as_str() {
1618            "done" => Status::Done,
1619            "in_progress" => Status::InProgress,
1620            "in_review" => Status::InReview,
1621            "blocked" => Status::Blocked,
1622            "routed" => Status::Routed, // Rerouted (agent/model already set)
1623            "new" => Status::New,       // Legacy reroute path
1624            _ => Status::NeedsReview,
1625        };
1626        // Store-first: update SQLite (must succeed), then mirror to backend.
1627        if let Some(ref store) = self.store
1628            && let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &task.id.0).await
1629        {
1630            store
1631                .update_status(store_id, tasks::status_to_task_status(new_status))
1632                .await
1633                .context("store-first status update failed in runner")?;
1634        }
1635        if let Err(e) = backend.update_status(&task.id, new_status).await {
1636            tracing::warn!(
1637                task_id,
1638                ?new_status,
1639                err = %e,
1640                "failed to mirror runner status to backend — store is authoritative"
1641            );
1642        }
1643
1644        let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ");
1645        let mut comment = if !summary.is_empty() {
1646            format!("[{now}] {status}: {summary}")
1647        } else if !last_error.is_empty() {
1648            format!("[{now}] {status}: {last_error}")
1649        } else {
1650            format!("[{now}] {status}")
1651        };
1652
1653        // Append visible attribution footer (matches PR body style)
1654        let footer = engine::attribution_footer("Created", &agent_name, model);
1655        comment.push_str(&footer);
1656
1657        backend.post_comment(&task.id, &comment).await?;
1658
1659        Ok(weight_signal)
1660    }
1661
1662    /// Process delegations from an agent response.
1663    ///
1664    /// Creates child GitHub issues for each delegation and marks the parent
1665    /// as blocked. The engine's Phase 4 unblock mechanism will re-activate
1666    /// the parent when all children are done.
1667    async fn process_delegations(
1668        &self,
1669        parent_task: &ExternalTask,
1670        delegations: &[Delegation],
1671        backend: &Arc<dyn ExternalBackend>,
1672    ) -> anyhow::Result<()> {
1673        let parent_id = &parent_task.id;
1674
1675        // Fetch existing open tasks once for dedup — prevents creating duplicate
1676        // GitHub issues when the same delegations are produced on repeated runs.
1677        let existing_titles: HashSet<String> = match backend.list_all_tasks().await {
1678            Ok(tasks) => tasks.into_iter().map(|t| t.title).collect(),
1679            Err(e) => {
1680                tracing::warn!(err = %e, "failed to fetch tasks for dedup — will create all");
1681                HashSet::new()
1682            }
1683        };
1684
1685        for delegation in delegations {
1686            // Skip if an issue with the same title already exists
1687            if existing_titles.contains(&delegation.title) {
1688                tracing::info!(
1689                    parent = parent_id.0,
1690                    title = delegation.title,
1691                    "skipping delegation — issue with same title already exists"
1692                );
1693                continue;
1694            }
1695
1696            // Build labels: status:new + any labels from the delegation
1697            let mut labels = delegation.labels.clone();
1698            labels.push("status:new".to_string());
1699
1700            // Build child body with delegation reference
1701            let child_body = format!(
1702                "{}\n\n---\n_Delegated from #{}_{}",
1703                delegation.body,
1704                parent_id.0,
1705                engine::orch_footer()
1706            );
1707
1708            match backend
1709                .create_sub_task(parent_id, &delegation.title, &child_body, &labels)
1710                .await
1711            {
1712                Ok(child_id) => {
1713                    tracing::info!(
1714                        parent = parent_id.0,
1715                        child = child_id.0,
1716                        title = delegation.title,
1717                        "created delegated subtask"
1718                    );
1719                }
1720                Err(e) => {
1721                    tracing::error!(
1722                        parent = parent_id.0,
1723                        title = delegation.title,
1724                        err = %e,
1725                        "failed to create delegated subtask"
1726                    );
1727                }
1728            }
1729        }
1730
1731        // Mark parent as blocked — store-first, then mirror to backend.
1732        if let Some(ref store) = self.store
1733            && let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &parent_id.0).await
1734        {
1735            store
1736                .update_status(store_id, TaskStatus::Blocked)
1737                .await
1738                .context("store-first: failed to block parent in store")?;
1739        }
1740        if let Err(e) = backend.update_status(parent_id, Status::Blocked).await {
1741            tracing::warn!(
1742                parent = parent_id.0,
1743                err = %e,
1744                "failed to mirror blocked status to backend — store is authoritative"
1745            );
1746        }
1747
1748        // Post summary comment on parent
1749        let summary = delegations
1750            .iter()
1751            .enumerate()
1752            .map(|(i, d)| format!("{}. {}", i + 1, d.title))
1753            .collect::<Vec<_>>()
1754            .join("\n");
1755
1756        backend
1757            .post_comment(
1758                parent_id,
1759                &format!(
1760                    "Delegated {} subtask(s):\n\n{}\n\nParent task is blocked until all subtasks complete.{}",
1761                    delegations.len(),
1762                    summary,
1763                    engine::orch_footer()
1764                ),
1765            )
1766            .await?;
1767
1768        Ok(())
1769    }
1770
1771    /// Resolve the project directory for this repo.
1772    pub(crate) async fn resolve_project_dir(&self) -> anyhow::Result<PathBuf> {
1773        // Explicit env var always wins
1774        if let Ok(dir) = env::var("PROJECT_DIR")
1775            && !dir.is_empty()
1776        {
1777            return Ok(PathBuf::from(dir));
1778        }
1779
1780        // Check projects list in global config — find path for this repo
1781        if let Ok(paths) = config::get_project_paths() {
1782            for path_str in &paths {
1783                let path = PathBuf::from(path_str);
1784                // Check if this project's .orch.yml has matching repo
1785                if let Ok(repo) = config::get_repo_for_project(&path) {
1786                    // Use tokio::fs::metadata for async-safe existence check
1787                    let path_for_check = path.clone();
1788                    if repo == self.repo && fs::metadata(&path_for_check).await.is_ok() {
1789                        return Ok(path);
1790                    }
1791                }
1792            }
1793        }
1794
1795        // Legacy: check config project_dir
1796        if let Ok(dir) = config::get("project_dir")
1797            && !dir.is_empty()
1798        {
1799            let path = PathBuf::from(&dir);
1800            let path_for_check = path.clone();
1801            if fs::metadata(&path_for_check).await.is_ok() {
1802                return Ok(path);
1803            }
1804        }
1805
1806        // Check for bare clone
1807        let parts: Vec<&str> = self.repo.split('/').collect();
1808        if parts.len() == 2 {
1809            let bare = self
1810                .orch_home
1811                .join("projects")
1812                .join(parts[0])
1813                .join(format!("{}.git", parts[1]));
1814            let bare_for_check = bare.clone();
1815            if task::spawn_blocking(move || bare_for_check.exists())
1816                .await
1817                .unwrap_or(false)
1818            {
1819                return Ok(bare);
1820            }
1821        }
1822
1823        // No resolvable project directory: not in active config, no legacy
1824        // project_dir, no bare clone. Falling back to the service's own CWD
1825        // here would silently point worktree creation at an unrelated
1826        // directory (see issue #3416) — surface a clear error instead.
1827        Err(NoProjectDir {
1828            repo: self.repo.clone(),
1829            bare_clone: self
1830                .orch_home
1831                .join("projects")
1832                .join(self.repo.split('/').next().unwrap_or_default())
1833                .join(format!(
1834                    "{}.git",
1835                    self.repo.split('/').nth(1).unwrap_or_default()
1836                )),
1837        }
1838        .into())
1839    }
1840}
1841
1842/// Update the GitHub agent label on an external task when it is rerouted to a new agent.
1843///
1844/// Removes the old `agent:<old_agent>` label, ensures the new `agent:<new_agent>` label exists
1845/// in the repo, then applies it. Returns `true` only when both `ensure_label` (repo label creation)
1846/// and `set_labels` (issue label assignment) succeed. `remove_label` failures are non-fatal.
1847async fn update_agent_label_after_reroute(
1848    #[cfg_attr(
1849        test,
1850        expect(unused_variables, reason = "only the label call outside tests uses it")
1851    )]
1852    repo: &str,
1853    task: &ExternalTask,
1854    backend: &Arc<dyn ExternalBackend>,
1855    old_agent: &str,
1856    new_agent: &str,
1857) -> bool {
1858    let old_label = format!("agent:{old_agent}");
1859    if let Err(e) = backend.remove_label(&task.id, &old_label).await {
1860        tracing::warn!(
1861            task_id = ?task.id,
1862            label = old_label,
1863            error = %e,
1864            "failed to remove old agent label during re-route"
1865        );
1866    }
1867    let new_label = format!("agent:{new_agent}");
1868
1869    // ensure_label creates the repo-level label if missing (best-effort).
1870    // Skipped under #[cfg(test)] to avoid real GitHub API calls.
1871    #[cfg(not(test))]
1872    {
1873        match GhHttp::new() {
1874            Ok(gh) => {
1875                if gh
1876                    .ensure_label(
1877                        repo,
1878                        &new_label,
1879                        http::status_label_color(&new_label),
1880                        &format!("Agent: {new_agent}"),
1881                    )
1882                    .await
1883                    .is_err()
1884                {
1885                    tracing::warn!(
1886                        task_id = ?task.id,
1887                        label = %new_label,
1888                        "ensure_label failed during agent label update after failover"
1889                    );
1890                }
1891            }
1892            Err(e) => {
1893                tracing::warn!(
1894                    task_id = ?task.id,
1895                    err = %e,
1896                    "GhHttp::new() failed — skipping ensure_label"
1897                );
1898            }
1899        }
1900    }
1901
1902    if let Err(e) = backend
1903        .set_labels(&task.id, slice::from_ref(&new_label))
1904        .await
1905    {
1906        tracing::warn!(
1907            task_id = ?task.id,
1908            label = %new_label,
1909            error = %e,
1910            "set_labels failed during agent label update after failover"
1911        );
1912        return false;
1913    }
1914    true
1915}
1916
1917/// Weight signal for a run that ended in `needs_review`. `prepare_task` clears `last_error` at run
1918/// start, so a non-empty value means `handle_error` ended this run
1919fn needs_review_signal(last_error: &str, agent: &str) -> WeightSignal {
1920    let agent = agent.to_string();
1921    if classify_run_error_type(last_error) == "rate_limit" {
1922        WeightSignal::RateLimited { agent }
1923    } else if !last_error.is_empty() {
1924        WeightSignal::Failed { agent }
1925    } else {
1926        WeightSignal::Success { agent }
1927    }
1928}
1929
1930#[cfg(test)]
1931mod tests {
1932    use super::*;
1933    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
1934    use crate::engine::runner::agents::patterns::safe_tail as safe_utf8_tail;
1935    use crate::parser::AgentResponse;
1936    use crate::store::{NewTask, TaskStatus, TaskStore};
1937    use async_trait::async_trait;
1938    use chrono::Utc;
1939    use std::sync::{Arc, Mutex};
1940    use tempfile::TempDir;
1941    use tokio::process::Command;
1942
1943    static ENV_LOCK: Mutex<()> = Mutex::new(());
1944
1945    fn ok_parse_result(status: &str) -> agents::ParsedResponse {
1946        agents::ParsedResponse {
1947            response: AgentResponse {
1948                status: status.to_string(),
1949                summary: String::new(),
1950                accomplished: vec![],
1951                remaining: vec![],
1952                files: vec![],
1953                error: None,
1954                input_tokens: None,
1955                output_tokens: None,
1956                learnings: vec![],
1957                delegations: vec![],
1958                screenshots: vec![],
1959                screenshot_note: None,
1960            },
1961            input_tokens: None,
1962            output_tokens: None,
1963            cost_usd: None,
1964            duration_ms: None,
1965        }
1966    }
1967
1968    // ── safe_utf8_tail ───────────────────────────────────────────────────────
1969
1970    #[test]
1971    fn safe_utf8_tail_short_string_returned_as_is() {
1972        assert_eq!(safe_utf8_tail("hello", 100), "hello");
1973        assert_eq!(safe_utf8_tail("", 10), "");
1974    }
1975
1976    #[test]
1977    fn safe_utf8_tail_truncates_ascii() {
1978        let s = "abcdefghij"; // 10 bytes
1979        assert_eq!(safe_utf8_tail(s, 5), "fghij");
1980        assert_eq!(safe_utf8_tail(s, 10), "abcdefghij");
1981    }
1982
1983    #[test]
1984    fn safe_utf8_tail_handles_multibyte_boundary() {
1985        // "日本語" is 3 chars × 3 bytes each = 9 bytes total
1986        let s = "日本語";
1987        assert_eq!(s.len(), 9);
1988        // Cutting at 8 bytes lands in the middle of "語" (3 bytes at 6..9).
1989        // safe_utf8_tail should walk forward to the next boundary at 9 (end) rather
1990        // than panicking on an invalid slice.
1991        let tail = safe_utf8_tail(s, 8);
1992        assert!(
1993            s.ends_with(tail),
1994            "tail must be a valid suffix of the original"
1995        );
1996        assert!(
1997            str::from_utf8(tail.as_bytes()).is_ok(),
1998            "tail must be valid UTF-8"
1999        );
2000    }
2001
2002    #[test]
2003    fn safe_utf8_tail_handles_mixed_ascii_and_multibyte() {
2004        let s = "hello日本語world"; // ascii + CJK + ascii
2005        for max_bytes in [1, 3, 5, 7, 11, 13, 20, 100] {
2006            let tail = safe_utf8_tail(s, max_bytes);
2007            assert!(
2008                str::from_utf8(tail.as_bytes()).is_ok(),
2009                "max_bytes={max_bytes}: tail must be valid UTF-8"
2010            );
2011            assert!(
2012                s.ends_with(tail),
2013                "max_bytes={max_bytes}: tail must be a suffix of original"
2014            );
2015        }
2016    }
2017
2018    #[test]
2019    fn safe_utf8_tail_exact_boundary() {
2020        let s = "abcdef"; // 6 bytes
2021        assert_eq!(safe_utf8_tail(s, 6), "abcdef"); // exactly fits
2022        assert_eq!(safe_utf8_tail(s, 0), ""); // empty tail
2023    }
2024
2025    #[test]
2026    fn parse_success_output_synthesizes_done_from_plain_text() {
2027        let runner = agents::get_runner("claude");
2028        let parsed = parse_success_output(
2029            "123",
2030            "claude",
2031            &*runner,
2032            "No open positions, no trade executions, no conditions change.",
2033        )
2034        .unwrap();
2035
2036        assert_eq!(parsed.response.status, "done");
2037        assert_eq!(
2038            parsed.response.summary,
2039            "No open positions, no trade executions, no conditions change."
2040        );
2041    }
2042
2043    #[test]
2044    fn parse_success_output_synthesizes_needs_review_from_error_text() {
2045        let runner = agents::get_runner("claude");
2046        let parsed =
2047            parse_success_output("123", "claude", &*runner, "Failed to update branch").unwrap();
2048
2049        assert_eq!(parsed.response.status, "needs_review");
2050        assert_eq!(
2051            parsed.response.error.as_deref(),
2052            Some("agent returned plain text instead of JSON")
2053        );
2054    }
2055
2056    #[test]
2057    fn parse_success_output_keeps_empty_text_invalid() {
2058        let runner = agents::get_runner("claude");
2059        let err = parse_success_output("123", "claude", &*runner, "").unwrap_err();
2060
2061        assert!(matches!(err, agents::AgentError::InvalidResponse { .. }));
2062    }
2063
2064    #[test]
2065    fn parse_success_output_preserves_tokens_from_ndjson() {
2066        // Simulates NDJSON output from --output-format stream-json where
2067        // the result event contains token usage. parse_response fails on
2068        // NDJSON (expects single JSON blob), but find_agent_result extracts
2069        // tokens from the NDJSON envelope so cost tracking is preserved.
2070        let ndjson = r#"{"type":"system","subtype":"init","session_id":"abc"}
2071{"type":"assistant","message":{"content":[{"type":"text","text":"Done."}],"usage":{"input_tokens":100,"output_tokens":50}}}
2072{"type":"result","subtype":"success","is_error":false,"duration_ms":5000,"result":"All changes committed successfully","usage":{"input_tokens":1234,"output_tokens":567}}"#;
2073        let runner = agents::get_runner("claude");
2074        let parsed = parse_success_output("456", "claude", &*runner, ndjson).unwrap();
2075
2076        assert_eq!(parsed.response.status, "done");
2077        assert_eq!(parsed.input_tokens, Some(1234));
2078        assert_eq!(parsed.output_tokens, Some(567));
2079        assert_eq!(parsed.duration_ms, Some(5000));
2080    }
2081
2082    #[test]
2083    fn parse_success_output_preserves_tokens_from_minimax_ndjson() {
2084        // MiniMax/Kimi use assistant event fallback when there's no result event.
2085        let ndjson = r#"{"type":"system","subtype":"init"}
2086{"type":"assistant","message":{"content":[{"type":"text","text":"No open positions."}],"usage":{"input_tokens":2000,"output_tokens":300}}}"#;
2087        let runner = agents::get_runner("minimax");
2088        let parsed = parse_success_output("789", "minimax", &*runner, ndjson).unwrap();
2089
2090        assert_eq!(parsed.response.status, "done");
2091        assert_eq!(parsed.input_tokens, Some(2000));
2092        assert_eq!(parsed.output_tokens, Some(300));
2093    }
2094
2095    #[test]
2096    fn classify_run_error_type_recognizes_specific_patterns() {
2097        let cases = [
2098            ("push failed: remote rejected", "push_failed"),
2099            ("No commits between main and branch", "no_commits"),
2100            ("create PR failed: 422", "pr_failed"),
2101            ("invalid response from agent", "parse_error"),
2102            ("parse error: missing json", "parse_error"),
2103            ("exceeded max attempts while rerouting", "max_attempts"),
2104            ("unit test failed", "ci_failure"),
2105            ("build failed: type mismatch", "ci_failure"),
2106            ("lint failed on warning", "ci_failure"),
2107            ("3 tests failed", "ci_failure"),
2108            ("test suite failed: 3 tests failed", "ci_failure"),
2109            (
2110                "claude interrupted: exited without a result envelope",
2111                "interrupted",
2112            ),
2113        ];
2114
2115        for (input, expected) in cases {
2116            assert_eq!(classify_run_error_type(input), expected, "input={input}");
2117        }
2118    }
2119
2120    #[test]
2121    fn classify_run_error_type_falls_back_for_unknown_errors() {
2122        assert_eq!(classify_run_error_type("something unexpected"), "failed");
2123    }
2124
2125    #[test]
2126    fn classify_run_error_type_connection_error_mentioning_pull_request_is_not_pr_failed() {
2127        // A generic GitHub API or connection error whose message happens to contain the
2128        // substring "pull request" must NOT be classified as pr_failed.
2129        assert_eq!(
2130            classify_run_error_type("failed to fetch pull request details from GitHub"),
2131            "failed"
2132        );
2133        assert_eq!(
2134            classify_run_error_type("connection reset while reading pull request data"),
2135            "failed"
2136        );
2137    }
2138
2139    #[test]
2140    fn classify_run_error_type_url_with_test_substring_is_not_ci_failure() {
2141        // A connection error whose URL path contains "/test-org/" must NOT be ci_failure.
2142        assert_eq!(
2143            classify_run_error_type(
2144                "failed to connect to api.github.com/repos/test-org/repo/pulls/123"
2145            ),
2146            "failed"
2147        );
2148        // Bare "test" word in an unrelated error must not trigger ci_failure.
2149        assert_eq!(
2150            classify_run_error_type("context deadline exceeded testing connection"),
2151            "failed"
2152        );
2153    }
2154
2155    #[test]
2156    fn classify_run_error_type_usage_without_limit_is_not_rate_limit() {
2157        // "usage" alone (e.g. token usage reporting) must NOT be rate_limit.
2158        assert_eq!(
2159            classify_run_error_type("token usage: 1245 tokens"),
2160            "failed"
2161        );
2162        assert_eq!(
2163            classify_run_error_type("incorrect usage of git command"),
2164            "failed"
2165        );
2166        // Only "usage limit" should trigger rate_limit.
2167        assert_eq!(
2168            classify_run_error_type("exceeded usage limit for this billing period"),
2169            "rate_limit"
2170        );
2171    }
2172
2173    #[test]
2174    fn classify_run_error_type_billing_cycle_is_not_auth_error() {
2175        // "billing cycle" quota-exhaustion messages must be classified as billing_cycle_exhausted,
2176        // not auth_error — aligning with cooldown.rs CreditExhaustionReason::BillingCycleExhausted.
2177        assert_eq!(
2178            classify_run_error_type(
2179                "You've reached your usage limit for this billing cycle. \
2180                 Your quota will be refreshed in the next cycle."
2181            ),
2182            "billing_cycle_exhausted"
2183        );
2184        assert_eq!(
2185            classify_run_error_type("quota exhausted for billing cycle"),
2186            "billing_cycle_exhausted"
2187        );
2188        // Plain "billing" without "cycle" that used to map to auth_error now falls through to "failed".
2189        assert_eq!(
2190            classify_run_error_type("billing account suspended"),
2191            "failed"
2192        );
2193    }
2194
2195    #[test]
2196    fn classify_run_error_type_monthly_quota_is_billing_cycle() {
2197        // Regression test for issue #3302: GitHub Copilot "You have exceeded your
2198        // monthly quota" must map to billing_cycle_exhausted, not "failed".
2199        assert_eq!(
2200            classify_run_error_type(
2201                "opencode failed: agent failed: You have exceeded your monthly quota"
2202            ),
2203            "billing_cycle_exhausted"
2204        );
2205        assert_eq!(
2206            classify_run_error_type("You have exceeded your monthly quota"),
2207            "billing_cycle_exhausted"
2208        );
2209    }
2210
2211    #[test]
2212    fn classify_run_outcome_routed_is_success() {
2213        // "routed" is a canonical non-success status (task will be re-dispatched).
2214        // It should be classified as "success" so the audit trail distinguishes it
2215        // from genuine failures (#2801).
2216        let parse_result = Ok(ok_parse_result("routed"));
2217        assert_eq!(
2218            classify_run_outcome("routed", &parse_result, false),
2219            "success"
2220        );
2221    }
2222
2223    #[test]
2224    fn classify_run_outcome_blocked_is_blocked() {
2225        let parse_result = Ok(ok_parse_result("blocked"));
2226        assert_eq!(
2227            classify_run_outcome("blocked", &parse_result, false),
2228            "blocked"
2229        );
2230    }
2231
2232    #[test]
2233    fn classify_run_outcome_non_canonical_is_failed() {
2234        // Non-canonical statuses should be classified as "failed" in the audit.
2235        // The explicit error message is provided by build_run_audit.
2236        let parse_result = Ok(ok_parse_result("fix_deployed"));
2237        assert_eq!(
2238            classify_run_outcome("fix_deployed", &parse_result, false),
2239            "failed"
2240        );
2241    }
2242
2243    #[test]
2244    fn classify_run_outcome_kimi_billing_cycle_is_not_rate_limit() {
2245        // Regression test for issue #3529: Kimi's exact billing-cycle exhaustion
2246        // message was stored as outcome="rate_limit" because the generic "usage
2247        // limit" pattern in detect_rate_limit() matched before credit-exhaustion
2248        // detection ever ran. The outcome must reflect the actual failure mode.
2249        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2250            Err(agents::AgentError::RateLimit {
2251                message: "Failed to authenticate. API Error: 403 You've reached your usage \
2252                          limit for this billing cycle. Your quota will be refreshed soon."
2253                    .to_string(),
2254            });
2255        assert_eq!(
2256            classify_run_outcome("needs_review", &parse_result, false),
2257            "billing_cycle_exhausted"
2258        );
2259    }
2260
2261    #[test]
2262    fn classify_run_outcome_generic_rate_limit_is_unaffected() {
2263        // A genuine rate limit (no billing-cycle/credit-exhaustion markers) must
2264        // still classify as "rate_limit".
2265        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2266            Err(agents::AgentError::RateLimit {
2267                message: "429 Too Many Requests".to_string(),
2268            });
2269        assert_eq!(
2270            classify_run_outcome("needs_review", &parse_result, false),
2271            "rate_limit"
2272        );
2273    }
2274
2275    #[test]
2276    fn classify_run_outcome_auth_billing_cycle_is_not_failed() {
2277        // Auth-classified billing-cycle exhaustion (e.g. "billing cycle exhausted"
2278        // matches detect_auth_error's pattern list) must also be distinguished from
2279        // a generic auth failure.
2280        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2281            Err(agents::AgentError::Auth {
2282                message: "billing cycle exhausted for this account".to_string(),
2283            });
2284        assert_eq!(
2285            classify_run_outcome("needs_review", &parse_result, false),
2286            "billing_cycle_exhausted"
2287        );
2288    }
2289
2290    #[test]
2291    fn classify_run_outcome_generic_auth_error_is_unaffected() {
2292        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2293            Err(agents::AgentError::Auth {
2294                message: "401 Unauthorized".to_string(),
2295            });
2296        assert_eq!(
2297            classify_run_outcome("needs_review", &parse_result, false),
2298            "failed"
2299        );
2300    }
2301
2302    #[test]
2303    fn classify_run_outcome_interrupted() {
2304        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2305            Err(agents::AgentError::Interrupted {
2306                message: "exited without a result envelope".to_string(),
2307            });
2308        assert_eq!(
2309            classify_run_outcome("new", &parse_result, false),
2310            "interrupted"
2311        );
2312    }
2313
2314    #[tokio::test]
2315    async fn build_run_audit_success_ignores_stale_last_error() {
2316        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2317        let task_id = store
2318            .create(&NewTask {
2319                repo: "owner/repo".to_string(),
2320                origin: "internal".to_string(),
2321                title: "Test".to_string(),
2322                ..Default::default()
2323            })
2324            .await
2325            .unwrap();
2326        store
2327            .set_fields(
2328                task_id,
2329                &[("last_error", serde_json::json!("no error info available"))],
2330            )
2331            .await
2332            .unwrap();
2333
2334        let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2335        let parse_result = Ok(ok_parse_result("done"));
2336        let started_at = Utc::now();
2337        let audit = runner
2338            .build_run_audit(RunAuditInput {
2339                task_id: &task_id.to_string(),
2340                status: "done",
2341                parse_result: &parse_result,
2342                raw_stdout: "",
2343                raw_stderr: "",
2344                started_at: &started_at,
2345                error_override: None,
2346                elapsed_secs: Some(1),
2347                push_failed: false,
2348            })
2349            .await;
2350
2351        assert_eq!(audit.outcome, "success");
2352        assert_eq!(audit.error, "");
2353    }
2354
2355    #[tokio::test]
2356    async fn reroute_clear_wipes_agent_when_it_still_matches_the_run() {
2357        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2358        let store_id = store
2359            .create(&NewTask {
2360                external_id: Some("42".to_string()),
2361                repo: "owner/repo".to_string(),
2362                origin: "github".to_string(),
2363                title: "Test".to_string(),
2364                ..Default::default()
2365            })
2366            .await
2367            .unwrap();
2368        store
2369            .update_status(store_id, TaskStatus::InProgress)
2370            .await
2371            .unwrap();
2372        store
2373            .set_fields(
2374                store_id,
2375                &[
2376                    ("agent", serde_json::json!("claude")),
2377                    ("model", serde_json::json!("sonnet")),
2378                ],
2379            )
2380            .await
2381            .unwrap();
2382
2383        let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store));
2384        runner
2385            .clear_agent_on_reroute_unless_fallback("42", "claude")
2386            .await;
2387
2388        let task = store.get(store_id).await.unwrap();
2389        assert!(
2390            task.agent.as_deref().unwrap_or("").is_empty(),
2391            "matching in-progress agent must be cleared for a normal reroute, got {:?}",
2392            task.agent
2393        );
2394        assert!(
2395            task.model.as_deref().unwrap_or("").is_empty(),
2396            "model must be cleared with the agent, got {:?}",
2397            task.model
2398        );
2399    }
2400
2401    #[tokio::test]
2402    async fn reroute_clear_keeps_a_different_in_progress_fallback() {
2403        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2404        let store_id = store
2405            .create(&NewTask {
2406                external_id: Some("42".to_string()),
2407                repo: "owner/repo".to_string(),
2408                origin: "github".to_string(),
2409                title: "Test".to_string(),
2410                ..Default::default()
2411            })
2412            .await
2413            .unwrap();
2414        store
2415            .update_status(store_id, TaskStatus::InProgress)
2416            .await
2417            .unwrap();
2418        store
2419            .set_fields(
2420                store_id,
2421                &[
2422                    ("agent", serde_json::json!("codex")),
2423                    ("model", serde_json::json!("")),
2424                ],
2425            )
2426            .await
2427            .unwrap();
2428
2429        let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store));
2430        runner
2431            .clear_agent_on_reroute_unless_fallback("42", "claude")
2432            .await;
2433
2434        let task = store.get(store_id).await.unwrap();
2435        assert_eq!(task.agent.as_deref(), Some("codex"));
2436        assert!(
2437            task.model.as_deref().unwrap_or("").is_empty(),
2438            "model stays empty so routing picks one for the fallback, got {:?}",
2439            task.model
2440        );
2441    }
2442
2443    #[tokio::test]
2444    async fn build_run_audit_ok_parse_prefers_parsed_error_over_stale_last_error() {
2445        // needs_review classifies as success and wipes the error. The #3851
2446        // misattribution is an Ok parse whose outcome is failed or blocked
2447        let cases = [("needs_action", "failed"), ("blocked", "blocked")];
2448        let parsed_error = "passbox Touch ID prompt timed out for cf/read-token";
2449        let stale = "minimax auth error: API Error: 402 insufficient balance (1008), clearing agent/model for re-route";
2450        for (status, expected_outcome) in cases {
2451            let store = Arc::new(TaskStore::open_memory().await.unwrap());
2452            let task_id = store
2453                .create(&NewTask {
2454                    repo: "owner/repo".to_string(),
2455                    origin: "internal".to_string(),
2456                    title: "Test".to_string(),
2457                    ..Default::default()
2458                })
2459                .await
2460                .unwrap();
2461            store
2462                .set_fields(task_id, &[("last_error", serde_json::json!(stale))])
2463                .await
2464                .unwrap();
2465
2466            let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2467            let parse_result = Ok(agents::ParsedResponse {
2468                response: AgentResponse {
2469                    status: status.to_string(),
2470                    summary: "cache_stats.py did not run".to_string(),
2471                    accomplished: vec![],
2472                    remaining: vec![],
2473                    files: vec![],
2474                    error: Some(parsed_error.to_string()),
2475                    input_tokens: None,
2476                    output_tokens: None,
2477                    learnings: vec![],
2478                    delegations: vec![],
2479                    screenshots: vec![],
2480                    screenshot_note: None,
2481                },
2482                input_tokens: None,
2483                output_tokens: None,
2484                cost_usd: None,
2485                duration_ms: None,
2486            });
2487            let started_at = Utc::now();
2488            let audit = runner
2489                .build_run_audit(RunAuditInput {
2490                    task_id: &task_id.to_string(),
2491                    status,
2492                    parse_result: &parse_result,
2493                    raw_stdout: "",
2494                    raw_stderr: "",
2495                    started_at: &started_at,
2496                    error_override: None,
2497                    elapsed_secs: Some(261),
2498                    push_failed: false,
2499                })
2500                .await;
2501
2502            assert_eq!(audit.outcome, expected_outcome, "status={status}");
2503            assert_eq!(audit.error, parsed_error, "status={status}");
2504        }
2505    }
2506
2507    #[tokio::test]
2508    async fn build_run_audit_non_success_uses_fallback_error_when_empty() {
2509        let runner = TaskRunner::new("owner/repo".to_string());
2510        let parse_result = Ok(ok_parse_result("new"));
2511        let started_at = Utc::now();
2512        let audit = runner
2513            .build_run_audit(RunAuditInput {
2514                task_id: "1",
2515                status: "new",
2516                parse_result: &parse_result,
2517                raw_stdout: "",
2518                raw_stderr: "",
2519                started_at: &started_at,
2520                error_override: None,
2521                elapsed_secs: Some(1),
2522                push_failed: false,
2523            })
2524            .await;
2525
2526        assert_eq!(audit.outcome, "failed");
2527        // "new" is a canonical non-success status — no error message needed.
2528        assert_eq!(audit.error, "");
2529    }
2530
2531    #[tokio::test]
2532    async fn build_run_audit_non_canonical_status_reports_explicit_error() {
2533        // Non-canonical statuses (not in the allowlist) with no parse error
2534        // should get an explicit "unrecognized status: X" message instead of
2535        // the opaque "no error info available" placeholder (#2801).
2536        let runner = TaskRunner::new("owner/repo".to_string());
2537        let parse_result = Ok(ok_parse_result("fix_deployed"));
2538        let started_at = Utc::now();
2539        let audit = runner
2540            .build_run_audit(RunAuditInput {
2541                task_id: "1",
2542                status: "fix_deployed",
2543                parse_result: &parse_result,
2544                raw_stdout: "",
2545                raw_stderr: "",
2546                started_at: &started_at,
2547                error_override: None,
2548                elapsed_secs: Some(1),
2549                push_failed: false,
2550            })
2551            .await;
2552
2553        assert_eq!(audit.outcome, "failed");
2554        assert_eq!(audit.error, "unrecognized status: fix_deployed");
2555    }
2556
2557    #[tokio::test]
2558    async fn build_run_audit_canonical_non_success_status_no_error_message() {
2559        // Canonical non-success statuses (e.g. "routed") should not produce
2560        // an error message — they're expected outcomes, not failures.
2561        for status in &["routed"] {
2562            let runner = TaskRunner::new("owner/repo".to_string());
2563            let parse_result = Ok(ok_parse_result(status));
2564            let started_at = Utc::now();
2565            let audit = runner
2566                .build_run_audit(RunAuditInput {
2567                    task_id: "1",
2568                    status,
2569                    parse_result: &parse_result,
2570                    raw_stdout: "",
2571                    raw_stderr: "",
2572                    started_at: &started_at,
2573                    error_override: None,
2574                    elapsed_secs: Some(1),
2575                    push_failed: false,
2576                })
2577                .await;
2578
2579            assert_eq!(audit.outcome, "success", "status={status}");
2580            assert_eq!(
2581                audit.error, "",
2582                "status={status}: no error for canonical status"
2583            );
2584        }
2585    }
2586
2587    #[tokio::test]
2588    async fn build_run_audit_blocked_uses_summary_when_error_missing() {
2589        let runner = TaskRunner::new("owner/repo".to_string());
2590        let parse_result = Ok(agents::ParsedResponse {
2591            response: AgentResponse {
2592                status: "blocked".to_string(),
2593                summary: "Could not proceed due to missing credentials".to_string(),
2594                accomplished: vec![],
2595                remaining: vec![],
2596                files: vec![],
2597                error: None,
2598                input_tokens: None,
2599                output_tokens: None,
2600                learnings: vec![],
2601                delegations: vec![],
2602                screenshots: vec![],
2603                screenshot_note: None,
2604            },
2605            input_tokens: None,
2606            output_tokens: None,
2607            cost_usd: None,
2608            duration_ms: None,
2609        });
2610        let started_at = Utc::now();
2611        let audit = runner
2612            .build_run_audit(RunAuditInput {
2613                task_id: "1",
2614                status: "blocked",
2615                parse_result: &parse_result,
2616                raw_stdout: "",
2617                raw_stderr: "",
2618                started_at: &started_at,
2619                error_override: None,
2620                elapsed_secs: Some(1),
2621                push_failed: false,
2622            })
2623            .await;
2624
2625        assert_eq!(audit.outcome, "blocked");
2626        assert_eq!(audit.error, "Could not proceed due to missing credentials");
2627    }
2628
2629    // ── resolve_project_dir ──────────────────────────────────────────────────
2630
2631    #[tokio::test]
2632    async fn resolve_project_dir_uses_project_dir_env() {
2633        // When PROJECT_DIR env var is set, it should always win.
2634        let runner = TaskRunner::new("owner/repo".to_string());
2635
2636        // Use a temp dir so the path exists
2637        let dir = tempfile::tempdir().unwrap();
2638        let dir_str = dir.path().to_string_lossy().to_string();
2639
2640        // Temporarily set the env var — note: tests run in parallel so we use a
2641        // dedicated key that only this test touches. std::env::set_var is not
2642        // thread-safe in general, but the subsequent call is synchronous and
2643        // isolated to the temp dir created above.
2644        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2645        unsafe { env::set_var("PROJECT_DIR", &dir_str) };
2646        let result = runner.resolve_project_dir().await;
2647        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2648        unsafe { env::remove_var("PROJECT_DIR") };
2649
2650        assert!(
2651            result.is_ok(),
2652            "resolve_project_dir should succeed with PROJECT_DIR set"
2653        );
2654        assert_eq!(result.unwrap(), dir.path());
2655    }
2656
2657    #[tokio::test]
2658    async fn resolve_project_dir_empty_project_dir_env_falls_through() {
2659        // An empty PROJECT_DIR should be ignored and fall through to other logic.
2660        let runner = TaskRunner::new("owner/testrepo-nonexistent".to_string());
2661        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2662        unsafe { env::set_var("PROJECT_DIR", "") };
2663        let result = runner.resolve_project_dir().await;
2664        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2665        unsafe { env::remove_var("PROJECT_DIR") };
2666
2667        // No active config entry, no legacy project_dir, no bare clone for this
2668        // repo: should return an explicit error rather than falling back to the
2669        // service's own CWD (see issue #3416).
2670        assert!(
2671            result.is_err(),
2672            "resolve_project_dir should error when no project dir can be resolved"
2673        );
2674        let err = result.unwrap_err().to_string();
2675        assert!(
2676            err.contains("owner/testrepo-nonexistent"),
2677            "error should name the unresolvable repo: {err}"
2678        );
2679    }
2680
2681    // ── mock backend for process_delegations ─────────────────────────────────
2682
2683    struct TrackingBackend {
2684        sub_tasks_created: Arc<Mutex<Vec<(String, String)>>>, // (title, parent_id)
2685        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
2686        comments: Arc<Mutex<Vec<(String, String)>>>, // (id, body)
2687    }
2688
2689    impl TrackingBackend {
2690        fn new() -> Self {
2691            Self {
2692                sub_tasks_created: Arc::new(Mutex::new(vec![])),
2693                status_updates: Arc::new(Mutex::new(vec![])),
2694                comments: Arc::new(Mutex::new(vec![])),
2695            }
2696        }
2697    }
2698
2699    #[async_trait]
2700    impl ExternalBackend for TrackingBackend {
2701        fn name(&self) -> &'static str {
2702            "tracking"
2703        }
2704        async fn create_task(
2705            &self,
2706            _t: &str,
2707            _b: &str,
2708            _l: &[String],
2709        ) -> anyhow::Result<ExternalId> {
2710            Ok(ExternalId("new".to_string()))
2711        }
2712        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2713            Ok(ExternalTask {
2714                id: id.clone(),
2715                title: String::new(),
2716                body: String::new(),
2717                state: "open".to_string(),
2718                labels: vec![],
2719                author: String::new(),
2720                created_at: "2026-01-01T00:00:00Z".to_string(),
2721                updated_at: "2026-01-01T00:00:00Z".to_string(),
2722                url: String::new(),
2723            })
2724        }
2725        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
2726            Ok(vec![])
2727        }
2728        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2729            Ok(vec![])
2730        }
2731        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2732            self.comments
2733                .lock()
2734                .unwrap()
2735                .push((id.0.clone(), body.to_string()));
2736            Ok(())
2737        }
2738        async fn set_labels(&self, _id: &ExternalId, _l: &[String]) -> anyhow::Result<()> {
2739            Ok(())
2740        }
2741        async fn remove_label(&self, _id: &ExternalId, _l: &str) -> anyhow::Result<()> {
2742            Ok(())
2743        }
2744        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2745            Ok(vec![])
2746        }
2747        async fn create_sub_task(
2748            &self,
2749            parent: &ExternalId,
2750            title: &str,
2751            _body: &str,
2752            _l: &[String],
2753        ) -> anyhow::Result<ExternalId> {
2754            self.sub_tasks_created
2755                .lock()
2756                .unwrap()
2757                .push((title.to_string(), parent.0.clone()));
2758            Ok(ExternalId(format!("child-{title}")))
2759        }
2760        async fn ensure_status_label(&self, _l: &str) -> anyhow::Result<()> {
2761            Ok(())
2762        }
2763        async fn has_open_issue_with_title(&self, _t: &str, _l: &str) -> anyhow::Result<bool> {
2764            Ok(false)
2765        }
2766        async fn health_check(&self) -> anyhow::Result<()> {
2767            Ok(())
2768        }
2769        async fn is_pr_merged(&self, _b: &str) -> anyhow::Result<bool> {
2770            Ok(false)
2771        }
2772        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2773            Ok(None)
2774        }
2775        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
2776            Ok(vec![])
2777        }
2778        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2779            self.status_updates
2780                .lock()
2781                .unwrap()
2782                .push((id.0.clone(), status));
2783            Ok(())
2784        }
2785    }
2786
2787    fn make_task(id: &str) -> ExternalTask {
2788        ExternalTask {
2789            id: ExternalId(id.to_string()),
2790            title: format!("Task {id}"),
2791            body: String::new(),
2792            state: "open".to_string(),
2793            labels: vec![],
2794            author: "bot".to_string(),
2795            created_at: "2026-01-01T00:00:00Z".to_string(),
2796            updated_at: "2026-01-01T00:00:00Z".to_string(),
2797            url: String::new(),
2798        }
2799    }
2800
2801    // ── process_delegations ───────────────────────────────────────────────────
2802
2803    #[expect(
2804        clippy::await_holding_lock,
2805        reason = "the env lock must stay held across the awaited calls"
2806    )]
2807    #[tokio::test]
2808    async fn process_delegations_creates_subtasks_and_blocks_parent() {
2809        let _guard = ENV_LOCK.lock().unwrap();
2810        let temp_home = TempDir::new().unwrap();
2811        let orch_home = temp_home.path().join(".orch");
2812        fs::create_dir_all(&orch_home).await.unwrap();
2813        let old_orch_home = env::var("ORCH_HOME").ok();
2814        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2815        unsafe { env::set_var("ORCH_HOME", &orch_home) };
2816
2817        let runner = TaskRunner::new("owner/repo".to_string());
2818        let parent = make_task("99");
2819        let backend = Arc::new(TrackingBackend::new());
2820        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<TrackingBackend>::clone(&backend);
2821
2822        let delegations = vec![
2823            Delegation {
2824                title: "Subtask A".to_string(),
2825                body: "Do A".to_string(),
2826                labels: vec!["enhancement".to_string()],
2827            },
2828            Delegation {
2829                title: "Subtask B".to_string(),
2830                body: "Do B".to_string(),
2831                labels: vec![],
2832            },
2833        ];
2834
2835        runner
2836            .process_delegations(&parent, &delegations, &backend_dyn)
2837            .await
2838            .unwrap();
2839
2840        // Two sub-tasks should have been created
2841        let created = backend.sub_tasks_created.lock().unwrap();
2842        assert_eq!(created.len(), 2);
2843        assert_eq!(created[0].1, "99", "parent id should be 99");
2844        assert!(
2845            created.iter().any(|(t, _)| t == "Subtask A"),
2846            "Subtask A should be created"
2847        );
2848        assert!(
2849            created.iter().any(|(t, _)| t == "Subtask B"),
2850            "Subtask B should be created"
2851        );
2852        drop(created);
2853
2854        // Parent should be marked blocked
2855        let updates = backend.status_updates.lock().unwrap();
2856        assert!(
2857            updates
2858                .iter()
2859                .any(|(id, s)| id == "99" && *s == Status::Blocked),
2860            "parent should be blocked"
2861        );
2862        drop(updates);
2863
2864        // A comment summarising the delegations should be posted
2865        let comments = backend.comments.lock().unwrap();
2866        assert!(
2867            !comments.is_empty(),
2868            "a delegation summary comment should be posted"
2869        );
2870        let comment_body = &comments[0].1;
2871        assert!(
2872            comment_body.contains("Subtask A"),
2873            "summary should mention Subtask A"
2874        );
2875        assert!(
2876            comment_body.contains("Subtask B"),
2877            "summary should mention Subtask B"
2878        );
2879
2880        if let Some(old) = old_orch_home {
2881            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2882            unsafe { env::set_var("ORCH_HOME", old) };
2883        } else {
2884            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2885            unsafe { env::remove_var("ORCH_HOME") };
2886        }
2887    }
2888
2889    #[expect(
2890        clippy::await_holding_lock,
2891        reason = "the env lock must stay held across the awaited calls"
2892    )]
2893    #[tokio::test]
2894    async fn process_delegations_single_subtask() {
2895        let _guard = ENV_LOCK.lock().unwrap();
2896        let temp_home = TempDir::new().unwrap();
2897        let orch_home = temp_home.path().join(".orch");
2898        fs::create_dir_all(&orch_home).await.unwrap();
2899        let old_orch_home = env::var("ORCH_HOME").ok();
2900        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2901        unsafe { env::set_var("ORCH_HOME", &orch_home) };
2902
2903        let runner = TaskRunner::new("owner/repo".to_string());
2904        let parent = make_task("101");
2905        let backend = Arc::new(TrackingBackend::new());
2906        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<TrackingBackend>::clone(&backend);
2907
2908        let delegations = vec![Delegation {
2909            title: "Only Child".to_string(),
2910            body: "Do only child".to_string(),
2911            labels: vec!["feature".to_string()],
2912        }];
2913
2914        runner
2915            .process_delegations(&parent, &delegations, &backend_dyn)
2916            .await
2917            .unwrap();
2918
2919        let created = backend.sub_tasks_created.lock().unwrap();
2920        assert_eq!(created.len(), 1);
2921        assert_eq!(created[0].0, "Only Child");
2922        drop(created);
2923
2924        let updates = backend.status_updates.lock().unwrap();
2925        assert!(
2926            updates
2927                .iter()
2928                .any(|(id, s)| id == "101" && *s == Status::Blocked),
2929            "parent should be blocked"
2930        );
2931        drop(updates);
2932
2933        let comments = backend.comments.lock().unwrap();
2934        let body = &comments[0].1;
2935        assert!(
2936            body.contains("1 subtask"),
2937            "comment should count one subtask"
2938        );
2939
2940        if let Some(old) = old_orch_home {
2941            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2942            unsafe { env::set_var("ORCH_HOME", old) };
2943        } else {
2944            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2945            unsafe { env::remove_var("ORCH_HOME") };
2946        }
2947    }
2948
2949    // ── weight signal logic ───────────────────────────────────────────────────
2950
2951    /// Test helper that mirrors the weight signal logic from `run_with_context`.
2952    /// Reroutes only emit `RateLimited` when the error type is genuinely "rate_limit".
2953    fn weight_signal_for(status: &str, error_type: &str, agent: &str) -> WeightSignal {
2954        let is_rerouted = status == "new" || status == "routed";
2955        let is_rate_limit_error = error_type == "rate_limit";
2956        if is_rerouted {
2957            if is_rate_limit_error {
2958                WeightSignal::RateLimited {
2959                    agent: agent.to_string(),
2960                }
2961            } else {
2962                WeightSignal::Rerouted
2963            }
2964        } else if status == "done"
2965            || status == "needs_review"
2966            || status == "in_progress"
2967            || status == "in_review"
2968        {
2969            WeightSignal::Success {
2970                agent: agent.to_string(),
2971            }
2972        } else if status == "blocked" {
2973            WeightSignal::Blocked
2974        } else {
2975            WeightSignal::None
2976        }
2977    }
2978
2979    #[test]
2980    fn weight_signal_success_includes_needs_review() {
2981        // The normal happy-path for code tasks: agent reports done, has a PR,
2982        // so handle_success returns "needs_review". This must map to Success.
2983        let signal = weight_signal_for("needs_review", "success", "claude");
2984        assert!(
2985            matches!(signal, WeightSignal::Success { agent } if agent == "claude"),
2986            "needs_review should produce WeightSignal::Success"
2987        );
2988    }
2989
2990    #[test]
2991    fn needs_review_signal_success_for_empty_error() {
2992        let signal = needs_review_signal("", "claude");
2993        assert!(matches!(signal, WeightSignal::Success { agent } if agent == "claude"));
2994    }
2995
2996    #[test]
2997    fn needs_review_signal_failed_after_timeout_with_no_fallback() {
2998        let signal =
2999            needs_review_signal("claude timed out after 1804s, no fallback agents", "claude");
3000        assert!(matches!(signal, WeightSignal::Failed { agent } if agent == "claude"));
3001    }
3002
3003    #[test]
3004    fn needs_review_signal_rate_limited_for_rate_limit() {
3005        let signal = needs_review_signal("rate limit exceeded, try again later", "claude");
3006        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3007    }
3008
3009    #[test]
3010    fn weight_signal_success_for_all_done_statuses() {
3011        for status in ["done", "needs_review", "in_progress", "in_review"] {
3012            let signal = weight_signal_for(status, "success", "codex");
3013            assert!(
3014                matches!(signal, WeightSignal::Success { agent } if agent == "codex"),
3015                "{status} should produce WeightSignal::Success"
3016            );
3017        }
3018    }
3019
3020    #[test]
3021    fn weight_signal_rate_limited_for_reroute_with_rate_limit_error() {
3022        // Reroute due to rate limit should emit RateLimited.
3023        let signal = weight_signal_for("new", "rate_limit", "claude");
3024        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3025    }
3026
3027    #[test]
3028    fn weight_signal_rate_limited_for_routed_status_with_rate_limit_error() {
3029        let signal = weight_signal_for("routed", "rate_limit", "claude");
3030        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3031    }
3032
3033    #[test]
3034    fn weight_signal_rerouted_for_non_rate_limit_reroute() {
3035        // Reroute due to silence detection, timeout, or other non-rate-limit errors
3036        // should emit Rerouted (not RateLimited and not None).
3037        // Rerouted maps to "new" in tick so the task stays retryable and is never
3038        // converted to needs_review → done via the no-code review shortcut.
3039        for error_type in ["timeout", "silence detected", "parse_error", "failed"] {
3040            for status in ["new", "routed"] {
3041                let signal = weight_signal_for(status, error_type, "claude");
3042                assert!(
3043                    matches!(signal, WeightSignal::Rerouted),
3044                    "{status} with error_type={error_type} should produce WeightSignal::Rerouted, not RateLimited or None"
3045                );
3046            }
3047        }
3048    }
3049
3050    #[test]
3051    fn silence_reset_rerouted_never_becomes_needs_review() {
3052        // Regression test for: silence detection sets task to new, runner returns
3053        // WeightSignal::Rerouted, tick must map that to "new" — not "needs_review".
3054        // If it mapped to needs_review the no-code shortcut would close the task as
3055        // done with no successful agent run in the audit trail.
3056        let signal = weight_signal_for("new", "silence detected", "claude");
3057        assert!(
3058            matches!(signal, WeightSignal::Rerouted),
3059            "silence-reset reroute must produce Rerouted, never None/needs_review"
3060        );
3061        // Verify tick mapping: Rerouted => "new" (not needs_review or done)
3062        let display_status = match &signal {
3063            WeightSignal::Rerouted => "new",
3064            WeightSignal::None => "needs_review", // the old (buggy) path
3065            _ => "other",
3066        };
3067        assert_eq!(
3068            display_status, "new",
3069            "Rerouted must map to 'new' in tick, not 'needs_review'"
3070        );
3071    }
3072
3073    #[test]
3074    fn weight_signal_blocked_status() {
3075        let signal = weight_signal_for("blocked", "success", "claude");
3076        assert!(matches!(signal, WeightSignal::Blocked));
3077    }
3078
3079    // ── parse_success_output token extraction ────────────────────────────────
3080
3081    /// Mock runner that always returns InvalidResponse with truncated text,
3082    /// simulating what minimax/codex do when they can't parse structured JSON.
3083    struct FailingMockRunner;
3084
3085    impl agents::AgentRunner for FailingMockRunner {
3086        #[cfg(test)]
3087        fn name(&self) -> &'static str {
3088            "opencode"
3089        }
3090
3091        fn stream_events(&self, _event: &serde_json::Value) -> Vec<agents::StreamEvent> {
3092            vec![]
3093        }
3094
3095        fn build_command(
3096            &self,
3097            _model: Option<&str>,
3098            _timeout_cmd: &str,
3099            _sys_file: &str,
3100            _msg_file: &str,
3101            _permissions: &agents::PermissionRules,
3102        ) -> String {
3103            String::new()
3104        }
3105
3106        fn parse_response(&self, raw: &str) -> Result<agents::ParsedResponse, agents::AgentError> {
3107            // Simulate what minimax does: truncate to 300 chars
3108            Err(agents::AgentError::InvalidResponse {
3109                raw: raw.chars().take(30).collect(),
3110            })
3111        }
3112
3113        fn extract_text(&self, raw: &str) -> Result<String, agents::AgentError> {
3114            Err(agents::AgentError::InvalidResponse {
3115                raw: raw.to_string(),
3116            })
3117        }
3118
3119        fn classify_error(
3120            &self,
3121            exit_code: i32,
3122            _stdout: &str,
3123            _stderr: &str,
3124        ) -> agents::AgentError {
3125            agents::AgentError::Unknown {
3126                exit_code,
3127                message: "mock error".into(),
3128            }
3129        }
3130
3131        fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
3132            anyhow::bail!("not implemented for mock")
3133        }
3134    }
3135
3136    #[test]
3137    fn parse_success_output_extracts_tokens_from_raw_stdout_not_error_raw() {
3138        // OpenCode NDJSON output with token metadata in step_finish
3139        let ndjson = r#"{"type":"text","timestamp":1,"part":{"type":"text","text":"Here is my response: {\"status\":\"done\",\"summary\":\"fixed\"}}"}}
3140{"type":"step_finish","timestamp":2,"part":{"type":"step-finish","reason":"stop","cost":0,"tokens":{"total":95177,"input":94920,"output":257}}}"#;
3141
3142        let runner = FailingMockRunner;
3143        let result = parse_success_output("1550", "opencode", &runner, ndjson);
3144
3145        // Should succeed (synthesized from text) with tokens preserved
3146        let parsed = result.expect("should synthesize response from text");
3147        assert_eq!(parsed.input_tokens, Some(94920));
3148        assert_eq!(parsed.output_tokens, Some(257));
3149    }
3150
3151    #[test]
3152    fn parse_success_output_no_tokens_when_ndjson_has_none() {
3153        // Plain text with an explicit completion signal but no token metadata.
3154        // Uses "No open positions" which is in the explicit_done list.
3155        let text = "No open positions, no trade executions, no conditions change.";
3156
3157        let runner = FailingMockRunner;
3158        let result = parse_success_output("1551", "opencode", &runner, text);
3159
3160        // Should synthesize a "done" response, but without tokens (no NDJSON envelope).
3161        let parsed = result.expect("should synthesize response from explicit-done text");
3162        assert_eq!(parsed.input_tokens, None);
3163        assert_eq!(parsed.output_tokens, None);
3164    }
3165
3166    #[test]
3167    fn parse_success_output_uses_agent_result_directly() {
3168        // Here we test that if find_agent_result succeeds and returns valid JSON, we use it directly
3169        // WITHOUT calling agent_runner.parse_response at all.
3170        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"status\":\"done\",\"summary\":\"my summary\",\"accomplished\":[],\"remaining\":[],\"files\":[]}"}"#;
3171
3172        // FailingMockRunner always returns InvalidResponse, so if we called it, we'd synthesize "done" instead of getting "my summary".
3173        let runner = FailingMockRunner;
3174        let result = parse_success_output("999", "claude", &runner, ndjson);
3175
3176        let parsed = result.expect("should parse successfully via find_agent_result");
3177        assert_eq!(parsed.response.status, "done");
3178        assert_eq!(parsed.response.summary, "my summary");
3179    }
3180
3181    #[test]
3182    fn parse_success_output_treats_agent_result_is_error_as_authoritative() {
3183        // Here we test that if find_agent_result returns is_error=true, we return AgentFailed
3184        // instead of falling back to synthesis or agent_runner.parse_response.
3185        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"I failed because of reasons"}"#;
3186
3187        let runner = FailingMockRunner;
3188        let result = parse_success_output("999", "claude", &runner, ndjson);
3189
3190        let err = result.expect_err("should return error");
3191        match err {
3192            agents::AgentError::AgentFailed { message } => {
3193                assert_eq!(message, "I failed because of reasons");
3194            }
3195            _ => panic!("expected AgentFailed, got {err:?}"),
3196        }
3197    }
3198
3199    #[test]
3200    fn parse_success_output_classifies_model_unavailable_from_is_error_result() {
3201        // Regression test for issue #3607: the is_error=true NDJSON path
3202        // (used by codex's find_codex_result, which stringifies its own
3203        // classified AgentError into result_text) must re-classify a
3204        // model-not-found message as ModelUnavailable so it gets
3205        // model-scoped cooldown treatment instead of degrading to a
3206        // generic AgentFailed, which triggers an agent-wide cooldown.
3207        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"model unavailable (gpt-5.5): Reconnecting... 2/5 (unexpected status 404 Not Found: The model `gpt-5.5` does not exist or you do not have access to it.)"}"#;
3208
3209        let runner = FailingMockRunner;
3210        let result = parse_success_output("999", "claude", &runner, ndjson);
3211
3212        let err = result.expect_err("should return error");
3213        match err {
3214            agents::AgentError::ModelUnavailable { model, .. } => {
3215                assert_eq!(model, "gpt-5.5");
3216            }
3217            _ => panic!("expected ModelUnavailable, got {err:?}"),
3218        }
3219    }
3220
3221    #[test]
3222    fn nonzero_exit_with_ndjson_success_is_parsed_as_success() {
3223        // Simulate an agent that exits with non-zero code but emits NDJSON result
3224        // with is_error=false and a JSON result payload. The runner should accept
3225        // this as success because the NDJSON result is authoritative.
3226        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1234,"result":"{\"status\":\"done\",\"summary\":\"All good\"}","usage":{"input_tokens":10,"output_tokens":5}}"#;
3227
3228        // Use a runner that would normally fail parsing to ensure we're using
3229        // the find_agent_result path. FailingMockRunner.parse_response returns
3230        // InvalidResponse, but parse_success_output should short-circuit.
3231        let runner = FailingMockRunner;
3232
3233        // Construct a fake SessionOutput
3234        let session_output = session::SessionOutput {
3235            exit_code: 1,
3236            raw_stdout: ndjson.to_string(),
3237            raw_stderr: String::new(),
3238            elapsed_secs: Some(1),
3239            sandbox_error: None,
3240        };
3241
3242        // Call the helper we added via the task runner logic by reusing the
3243        // parse_success_output directly since it encapsulates the same behavior.
3244        let parsed =
3245            parse_success_output("task-ndjson", "claude", &runner, &session_output.raw_stdout)
3246                .expect("should parse NDJSON success");
3247        assert_eq!(parsed.response.status, "done");
3248        assert_eq!(parsed.response.summary, "All good");
3249        assert_eq!(parsed.input_tokens, Some(10));
3250        assert_eq!(parsed.output_tokens, Some(5));
3251    }
3252
3253    // ── agent label update after reroute ─────────────────────────────────────
3254
3255    /// Backend that wraps TrackingBackend but can be configured to fail label ops.
3256    struct LabelFailingBackend {
3257        inner: TrackingBackend,
3258        /// If set, set_labels returns this error.
3259        set_labels_err: Option<String>,
3260    }
3261
3262    impl LabelFailingBackend {
3263        fn new() -> Self {
3264            Self {
3265                inner: TrackingBackend::new(),
3266                set_labels_err: None,
3267            }
3268        }
3269        fn with_set_labels_err(mut self, err: impl Into<String>) -> Self {
3270            self.set_labels_err = Some(err.into());
3271            self
3272        }
3273    }
3274
3275    impl Default for LabelFailingBackend {
3276        fn default() -> Self {
3277            Self::new()
3278        }
3279    }
3280
3281    #[async_trait]
3282    impl ExternalBackend for LabelFailingBackend {
3283        fn name(&self) -> &'static str {
3284            "label_failing"
3285        }
3286        async fn create_task(
3287            &self,
3288            _t: &str,
3289            _b: &str,
3290            _l: &[String],
3291        ) -> anyhow::Result<ExternalId> {
3292            Ok(ExternalId("new".to_string()))
3293        }
3294        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3295            self.inner.get_task(id).await
3296        }
3297        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
3298            Ok(vec![])
3299        }
3300        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3301            Ok(vec![])
3302        }
3303        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3304            self.inner.post_comment(id, body).await
3305        }
3306        async fn set_labels(&self, id: &ExternalId, labels: &[String]) -> anyhow::Result<()> {
3307            if let Some(ref err) = self.set_labels_err {
3308                return Err(anyhow::anyhow!("{err}"));
3309            }
3310            self.inner.set_labels(id, labels).await
3311        }
3312        async fn remove_label(&self, id: &ExternalId, label: &str) -> anyhow::Result<()> {
3313            self.inner.remove_label(id, label).await
3314        }
3315        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3316            Ok(vec![])
3317        }
3318        async fn create_sub_task(
3319            &self,
3320            parent: &ExternalId,
3321            title: &str,
3322            body: &str,
3323            labels: &[String],
3324        ) -> anyhow::Result<ExternalId> {
3325            self.inner
3326                .create_sub_task(parent, title, body, labels)
3327                .await
3328        }
3329        async fn ensure_status_label(&self, label: &str) -> anyhow::Result<()> {
3330            self.inner.ensure_status_label(label).await
3331        }
3332        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3333            self.inner.update_status(id, status).await
3334        }
3335        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3336            self.inner.get_authenticated_user().await
3337        }
3338        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
3339            Ok(vec![])
3340        }
3341        async fn health_check(&self) -> anyhow::Result<()> {
3342            self.inner.health_check().await
3343        }
3344    }
3345
3346    /// `update_agent_label_after_reroute` returns `false` when `set_labels` fails.
3347    ///
3348    /// This tests the extracted helper directly, bypassing the full `run_with_context`
3349    /// path (which would fail earlier on worktree setup in a unit-test environment).
3350    #[tokio::test]
3351    async fn update_agent_label_returns_false_on_set_labels_failure() {
3352        let task = ExternalTask {
3353            id: ExternalId("test-reroute-1".to_string()),
3354            title: "Test".to_string(),
3355            body: "body".to_string(),
3356            state: "open".to_string(),
3357            labels: vec![],
3358            author: "bot".to_string(),
3359            created_at: "2026-01-01T00:00:00Z".to_string(),
3360            updated_at: "2026-01-01T00:00:00Z".to_string(),
3361            url: String::new(),
3362        };
3363        let backend: Arc<dyn ExternalBackend> =
3364            Arc::new(LabelFailingBackend::new().with_set_labels_err("network error"));
3365
3366        let result =
3367            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
3368                .await;
3369
3370        assert!(
3371            !result,
3372            "update_agent_label_after_reroute should return false when set_labels fails"
3373        );
3374    }
3375
3376    /// `update_agent_label_after_reroute` returns `true` when `set_labels` succeeds.
3377    #[tokio::test]
3378    async fn update_agent_label_returns_true_on_success() {
3379        let task = ExternalTask {
3380            id: ExternalId("test-reroute-2".to_string()),
3381            title: "Test".to_string(),
3382            body: "body".to_string(),
3383            state: "open".to_string(),
3384            labels: vec![],
3385            author: "bot".to_string(),
3386            created_at: "2026-01-01T00:00:00Z".to_string(),
3387            updated_at: "2026-01-01T00:00:00Z".to_string(),
3388            url: String::new(),
3389        };
3390        // LabelFailingBackend with no injected error delegates to TrackingBackend (always Ok).
3391        let backend: Arc<dyn ExternalBackend> = Arc::new(LabelFailingBackend::new());
3392
3393        let result =
3394            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
3395                .await;
3396
3397        // ensure_label is skipped in #[cfg(test)], set_labels succeeds via mock.
3398        assert!(
3399            result,
3400            "update_agent_label_after_reroute should return true when set_labels succeeds"
3401        );
3402    }
3403
3404    // ── NDJSON envelope fallback (issue #3078) ────────────────────────────────
3405
3406    /// Pattern A: agent emits domain JSON as result (no `status` field).
3407    /// NDJSON envelope says is_error=false, subtype=success.
3408    /// Was previously returning InvalidResponse → task re-ran.
3409    #[test]
3410    fn parse_success_output_ndjson_envelope_fallback_domain_json_result() {
3411        let domain_result = r#"{"file_path":"md/journal/2026-05-08-morning-briefing.md","summary":"Morning briefing filed","health":{"status":"ok"},"priorities":[]}"#;
3412        let ndjson = format!(
3413            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{domain_result:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
3414        );
3415
3416        let runner = agents::get_runner("glm");
3417        let parsed = parse_success_output("149200", "glm", &*runner, &ndjson)
3418            .expect("should synthesize done from NDJSON envelope when result is domain JSON");
3419
3420        assert_eq!(parsed.response.status, "done");
3421        assert!(
3422            parsed.response.summary.contains("file_path") || !parsed.response.summary.is_empty(),
3423            "summary should be populated from domain JSON result text"
3424        );
3425        assert_eq!(parsed.input_tokens, Some(3000));
3426        assert_eq!(parsed.output_tokens, Some(400));
3427    }
3428
3429    /// Grok-style envelope: no terminal_reason, subtype=success.
3430    #[test]
3431    fn parse_success_output_envelope_fallback_without_terminal_reason() {
3432        let receipt = r#"{"ok":true,"report":"notes.md","commit":"abc"}"#;
3433        let ndjson = format!(
3434            r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","result":{receipt:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
3435        );
3436        let runner = agents::get_runner("grok");
3437        let parsed = parse_success_output("1", "grok", &*runner, &ndjson).expect("done");
3438        assert_eq!(parsed.response.status, "done");
3439        assert_eq!(parsed.input_tokens, Some(3000));
3440    }
3441
3442    /// Pattern B: agent emits plain prose as result, no completion keywords.
3443    /// NDJSON envelope says is_error=false, subtype=success.
3444    /// Was previously returning InvalidResponse → task re-ran.
3445    #[test]
3446    fn parse_success_output_ndjson_envelope_fallback_plain_prose_result() {
3447        let prose = "PR #1008 now has zero diff vs main. The reverted ledger changes are gone and the branch is clean.";
3448        let ndjson = format!(
3449            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{prose:?},"usage":{{"input_tokens":2500,"output_tokens":300}}}}"#
3450        );
3451
3452        let runner = agents::get_runner("claude");
3453        let parsed = parse_success_output("149164", "claude", &*runner, &ndjson)
3454            .expect("should synthesize done from NDJSON envelope when result is plain prose");
3455
3456        assert_eq!(parsed.response.status, "done");
3457        assert_eq!(parsed.response.summary, prose);
3458        assert_eq!(parsed.input_tokens, Some(2500));
3459        assert_eq!(parsed.output_tokens, Some(300));
3460    }
3461
3462    /// Verify that is_error=true in NDJSON envelope still returns AgentFailed,
3463    /// not "done" — the envelope fallback must not override explicit errors.
3464    #[test]
3465    fn parse_success_output_ndjson_envelope_fallback_skipped_when_is_error_true() {
3466        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"result":"Something went wrong with no status"}"#;
3467
3468        let runner = agents::get_runner("claude");
3469        let err = parse_success_output("149999", "claude", &*runner, ndjson)
3470            .expect_err("is_error=true should return AgentFailed");
3471
3472        assert!(matches!(err, agents::AgentError::AgentFailed { .. }));
3473    }
3474
3475    fn tool_pair(id: &str, name: &str, cmd: &str, is_error: bool) -> String {
3476        format!(
3477            "{}\n{}\n",
3478            serde_json::json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":id,"name":name,"input":{"command":cmd}}]}}),
3479            serde_json::json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":id,"is_error":is_error,"content":"x"}]}}),
3480        )
3481    }
3482
3483    #[test]
3484    fn salvage_wrapup_accepts_commit_then_cleanup() {
3485        let raw = format!(
3486            "{}{}",
3487            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3488            tool_pair("2", "run_terminal_cmd", "git status", false)
3489        );
3490        assert!(salvage_wrapup(&raw).is_some());
3491    }
3492
3493    #[test]
3494    fn salvage_wrapup_rejects_edit_after_commit_or_unanswered_call() {
3495        let edit = format!(
3496            "{}{}",
3497            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3498            tool_pair("2", "search_replace", "", false)
3499        );
3500        assert!(salvage_wrapup(&edit).is_none());
3501        let pending = format!(
3502            "{}{}",
3503            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3504            serde_json::json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":"2","name":"run_terminal_cmd","input":{"command":"cargo test"}}]}})
3505        );
3506        assert!(salvage_wrapup(&pending).is_none());
3507        assert!(salvage_wrapup(&tool_pair("1", "run_terminal_cmd", "ls", false)).is_none());
3508    }
3509
3510    // ── terminal_reason:completed + exit 1 detection (issue #3087) ─────────────
3511
3512    /// kimi/minimax sometimes emit valid NDJSON with terminal_reason=completed
3513    /// but exit 1. In that case parse_success_output may fail (e.g. empty result
3514    /// text that synthesize can't parse), then parse_session_output would
3515    /// normally fall back to classify_error — grabbing a garbled JSON tail and
3516    /// producing a spurious Unknown/Auth error. The fix intercepts this and
3517    /// returns InvalidResponse (soft error, re-route) instead.
3518    #[test]
3519    fn parse_session_output_exit0_without_envelope_is_interrupted() {
3520        let session = session::SessionOutput {
3521            exit_code: 0,
3522            raw_stdout: "\u{4}\u{4}\u{4} truncated pane junk".to_string(),
3523            raw_stderr: String::new(),
3524            elapsed_secs: Some(180),
3525            sandbox_error: None,
3526        };
3527
3528        let runner = agents::get_runner("claude");
3529        let err = parse_session_output("3876", "claude", &*runner, &session)
3530            .expect_err("missing envelope should be an error");
3531        assert!(
3532            matches!(err, agents::AgentError::Interrupted { .. }),
3533            "exit 0 with no result envelope should be Interrupted, got {err:?}"
3534        );
3535    }
3536
3537    #[test]
3538    fn parse_session_output_exit1_with_terminal_reason_completed_returns_invalid_response() {
3539        // Empty result text causes synthesize_response_from_text to return None
3540        // (empty/whitespace-only input), so parse_success_output returns
3541        // InvalidResponse. With the fix, parse_session_output intercepts the
3542        // "terminal_reason":"completed" signal before classify_error and
3543        // returns InvalidResponse (re-route) instead of a garbled error.
3544        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":"","usage":{"input_tokens":1000,"output_tokens":100}}"#;
3545        let session = session::SessionOutput {
3546            exit_code: 1,
3547            raw_stdout: ndjson.to_string(),
3548            raw_stderr: String::new(),
3549            elapsed_secs: Some(30),
3550            sandbox_error: None,
3551        };
3552
3553        let runner = agents::get_runner("claude");
3554        let result = parse_session_output("149298", "claude", &*runner, &session);
3555
3556        let err = result.expect_err("should be an error, not Ok");
3557        assert!(
3558            matches!(err, agents::AgentError::InvalidResponse { .. }),
3559            "should be InvalidResponse, got {err:?}"
3560        );
3561    }
3562
3563    // ── GLM cost-telemetry exit-1 detection (issue #3094) ──────────────────────
3564
3565    /// GLM exits with code 1 but stdout is only cost telemetry JSON (contains
3566    /// "costUSD":). This must return InvalidResponse (soft re-route) instead of
3567    /// a hard Failed classification with spurious agent cooldown.
3568    #[test]
3569    fn parse_session_output_exit1_with_cost_telemetry_returns_invalid_response() {
3570        // GLM emits cost/usage JSON on clean exit; exit code 1 is a false failure.
3571        let telemetry = r#"{"type":"result","subtype":"success","is_error":false,"result":"","costUSD":0.042,"usage":{"input_tokens":5000,"output_tokens":800}}"#;
3572        let session = session::SessionOutput {
3573            exit_code: 1,
3574            raw_stdout: telemetry.to_string(),
3575            raw_stderr: String::new(),
3576            elapsed_secs: Some(120),
3577            sandbox_error: None,
3578        };
3579
3580        let runner = agents::get_runner("glm");
3581        let result = parse_session_output("149347", "glm", &*runner, &session);
3582
3583        let err = result.expect_err("should be an error, not Ok");
3584        assert!(
3585            matches!(err, agents::AgentError::InvalidResponse { .. }),
3586            "glm cost-telemetry exit-1 should return InvalidResponse, got {err:?}"
3587        );
3588    }
3589
3590    /// GLM with costUSD AND is_error:true — must propagate AgentFailed, not
3591    /// silently swallow the error via the cost-telemetry guard.
3592    #[test]
3593    fn parse_session_output_exit1_cost_telemetry_with_is_error_true_propagates_failure() {
3594        let telemetry = r#"{"type":"result","subtype":"error","is_error":true,"result":"tool failed","costUSD":0.01,"usage":{"input_tokens":100,"output_tokens":10}}"#;
3595        let session = session::SessionOutput {
3596            exit_code: 1,
3597            raw_stdout: telemetry.to_string(),
3598            raw_stderr: String::new(),
3599            elapsed_secs: Some(15),
3600            sandbox_error: None,
3601        };
3602
3603        let runner = agents::get_runner("glm");
3604        let result = parse_session_output("149348", "glm", &*runner, &session);
3605
3606        let err = result.expect_err("should be an error");
3607        assert!(
3608            matches!(err, agents::AgentError::AgentFailed { .. }),
3609            "is_error=true with costUSD must still return AgentFailed, got {err:?}"
3610        );
3611    }
3612
3613    /// terminal_reason=completed must NOT override actual error conditions.
3614    /// When is_error=true in the NDJSON envelope, AgentFailed must be returned
3615    /// even if terminal_reason=completed is also present. Only kimi/minimax
3616    /// agents use find_claude_result which extracts is_error, so use "kimi"
3617    /// as agent_name.
3618    #[test]
3619    fn parse_session_output_exit1_preserves_error_when_is_error_true() {
3620        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"terminal_reason":"completed","result":"failed"}"#;
3621        let session = session::SessionOutput {
3622            exit_code: 1,
3623            raw_stdout: ndjson.to_string(),
3624            raw_stderr: String::new(),
3625            elapsed_secs: Some(30),
3626            sandbox_error: None,
3627        };
3628
3629        let runner = agents::get_runner("kimi");
3630        let result = parse_session_output("149299", "kimi", &*runner, &session);
3631
3632        let err = result.expect_err("should be an error");
3633        assert!(
3634            matches!(err, agents::AgentError::AgentFailed { .. }),
3635            "is_error=true should return AgentFailed, got {err:?}"
3636        );
3637    }
3638
3639    /// RateLimit with terminal_reason=completed must be propagated as RateLimit,
3640    /// not downgraded to InvalidResponse. This covers claude session-limit errors
3641    /// that include "terminal_reason":"completed" in their NDJSON envelope — the
3642    /// terminal_reason guard was designed for kimi/minimax/glm false-positive
3643    /// exit codes but was intercepting classified errors like RateLimit.
3644    #[test]
3645    fn parse_session_output_exit1_with_rate_limit_and_terminal_reason_completed() {
3646        let ndjson = r#"{"type":"result","is_error":true,"terminal_reason":"completed","result":"You've hit your session limit · resets 2:20pm (America/Sao_Paulo)","usage":{"input_tokens":0,"output_tokens":0}}"#;
3647        let session = session::SessionOutput {
3648            exit_code: 1,
3649            raw_stdout: ndjson.to_string(),
3650            raw_stderr: String::new(),
3651            elapsed_secs: Some(120),
3652            sandbox_error: None,
3653        };
3654
3655        let runner = agents::get_runner("claude");
3656        let result = parse_session_output("152895", "claude", &*runner, &session);
3657
3658        let err = result.expect_err("rate-limit should be an error");
3659        assert!(
3660            matches!(err, agents::AgentError::RateLimit { .. }),
3661            "session limit should return RateLimit, not downgraded, got {err:?}"
3662        );
3663    }
3664}