Skip to main content

orch/engine/runner/
mod.rs

1//! Task runner — executes tasks using AI agents.
2//!
3//! Replaces `run_task.sh` with pure Rust. The runner:
4//! 1. Sets up a git worktree for isolation
5//! 2. Builds context (project instructions, skills, comments)
6//! 3. Renders prompts and spawns the agent in tmux
7//! 4. Waits for completion, collects and classifies the response
8//! 5. Handles errors (reroute on limits, fallback agents)
9//! 6. Auto-commits, pushes, and creates PRs
10//! 7. Updates status labels and posts result comments
11//!
12//! Submodules handle the heavy lifting:
13//! - [`task_init`] — guard checks, worktree setup, and invocation building
14//! - [`session`] — tmux session lifecycle and output collection
15//! - [`response_handler`] — success path: commit, push, PR, token storage
16//! - [`fallback`] — error classification and recovery strategies
17
18pub mod agent;
19pub mod agents;
20pub mod commit;
21pub mod context;
22pub mod cow;
23pub mod diff;
24pub mod direct;
25pub mod fallback;
26pub mod git_ops;
27#[cfg(test)]
28mod recorded_fixtures;
29pub mod response;
30pub mod response_handler;
31pub mod sandbox;
32pub mod session;
33pub mod task_init;
34pub mod worktree;
35
36use crate::backends::{ExternalBackend, ExternalTask, Status};
37use crate::config;
38use crate::engine;
39use crate::engine::cleanup;
40use crate::engine::cooldown;
41use crate::engine::router::RouteResult;
42use crate::engine::tasks;
43use crate::engine::tasks::is_internal_id;
44#[cfg(not(test))]
45use crate::github::http;
46use crate::github::http::GhHttp;
47use crate::home;
48use crate::parser;
49use crate::parser::AgentResponse;
50use crate::parser::Delegation;
51use crate::status_heuristics;
52use crate::store;
53use crate::store::InsertTaskMetric;
54use crate::store::RunTokenUsage;
55use crate::store::TaskStatus;
56use crate::store::TaskStore;
57use crate::tmux::TmuxManager;
58use anyhow::Context;
59use chrono::Utc;
60pub use response::WeightSignal;
61use serde_json::Value;
62use std::collections::HashSet;
63use std::env;
64use std::error;
65use std::fmt;
66use std::fmt::Debug;
67use std::fmt::Formatter;
68use std::path::PathBuf;
69use std::slice;
70use std::sync::Arc;
71use tokio::fs;
72use tokio::task;
73
74/// Fully materialized task-run audit data.
75#[derive(Debug, Clone)]
76pub struct RunAudit {
77    pub stdout: String,
78    pub stderr: String,
79    pub parsed_response: String,
80    pub outcome: String,
81    pub error: String,
82    pub input_tokens: u64,
83    pub output_tokens: u64,
84    pub total_cost_usd: f64,
85    pub duration_secs: f64,
86}
87
88/// Result returned by `run()` so callers can persist the audit trail.
89#[derive(Debug, Clone)]
90pub struct RunExecution {
91    pub status: String,
92    pub exit_code: Option<i32>,
93    pub audit: RunAudit,
94}
95
96fn serialize_parsed_response(
97    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
98    raw_stdout: &str,
99) -> String {
100    match parse_result {
101        Ok(parsed) => serde_json::json!({
102            "response": parsed.response,
103            "input_tokens": parsed.input_tokens,
104            "output_tokens": parsed.output_tokens,
105            "cost_usd": parsed.cost_usd,
106            "duration_ms": parsed.duration_ms,
107        })
108        .to_string(),
109        Err(agents::AgentError::InvalidResponse { raw }) => {
110            serde_json::json!({"raw": raw}).to_string()
111        }
112        Err(_) => serde_json::json!({"raw": raw_stdout}).to_string(),
113    }
114}
115
116fn classify_run_outcome(
117    status: &str,
118    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
119    push_failed: bool,
120) -> &'static str {
121    if push_failed {
122        return "push_failed";
123    }
124    match parse_result {
125        Err(agents::AgentError::Timeout { .. }) => "timeout",
126        Err(agents::AgentError::Interrupted { .. }) => "interrupted",
127        // Generic rate-limit patterns (e.g. "usage limit") also match billing-cycle
128        // exhaustion messages, so consult the credit-exhaustion detector first —
129        // otherwise Kimi-style "usage limit for this billing cycle" errors are
130        // stored as a plain "rate_limit" outcome (issue #3529).
131        Err(agents::AgentError::RateLimit { message }) => {
132            cooldown::detect_credit_exhaustion_outcome(message).unwrap_or("rate_limit")
133        }
134        Err(agents::AgentError::Auth { message }) => {
135            cooldown::detect_credit_exhaustion_outcome(message).unwrap_or("failed")
136        }
137        Err(agents::AgentError::InvalidResponse { .. }) => "parse_error",
138        Ok(_)
139            if matches!(
140                status,
141                "done" | "completed" | "in_progress" | "in_review" | "needs_review" | "routed"
142            ) =>
143        {
144            "success"
145        }
146        Ok(_) if status == "blocked" => "blocked",
147        // Non-canonical status that clearly reads as completion — success, not
148        // "unrecognized status" (#3627).
149        Ok(_) if status_heuristics::status_looks_like_descriptive_completion(status) => "success",
150        Err(_) | Ok(_) => "failed",
151    }
152}
153
154fn classify_run_error_type(last_error: &str) -> &'static str {
155    if last_error.is_empty() {
156        // No error: agent completed successfully and created a PR waiting for review
157        "success"
158    } else if last_error.contains("timeout") {
159        "timeout"
160    } else if last_error.contains("interrupted") {
161        "interrupted"
162    } else if last_error.contains("billing cycle") || last_error.contains("monthly quota") {
163        "billing_cycle_exhausted"
164    } else if last_error.contains("rate limit") || last_error.contains("usage limit") {
165        "rate_limit"
166    } else if last_error.contains("auth") {
167        "auth_error"
168    } else if last_error.contains("push failed") {
169        "push_failed"
170    } else if last_error.contains("No commits between") || last_error.contains("no commits") {
171        "no_commits"
172    } else if last_error.contains("create PR failed")
173        || last_error.contains("failed to create pull request")
174        || last_error.contains("pull request creation failed")
175    {
176        "pr_failed"
177    } else if last_error.contains("invalid response") || last_error.contains("parse error") {
178        "parse_error"
179    } else if last_error.contains("exceeded max attempts") {
180        "max_attempts"
181    } else {
182        let lower = last_error.to_ascii_lowercase();
183        if lower.contains("build failed")
184            || lower.contains("tests failed")
185            || lower.contains("test failed")
186            || lower.contains("test suite failed")
187            || lower.contains("lint failed")
188        {
189            "ci_failure"
190        } else {
191            "failed"
192        }
193    }
194}
195
196fn extract_run_tokens(
197    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
198) -> (u64, u64) {
199    match parse_result {
200        Ok(parsed) => (
201            parsed
202                .input_tokens
203                .or(parsed.response.input_tokens)
204                .unwrap_or(0),
205            parsed
206                .output_tokens
207                .or(parsed.response.output_tokens)
208                .unwrap_or(0),
209        ),
210        Err(_) => (0, 0),
211    }
212}
213
214#[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
215fn run_duration_secs(started_at: &chrono::DateTime<Utc>) -> f64 {
216    (Utc::now() - *started_at).num_milliseconds() as f64 / 1000.0
217}
218
219struct RunAuditInput<'a> {
220    task_id: &'a str,
221    status: &'a str,
222    parse_result: &'a Result<agents::ParsedResponse, agents::AgentError>,
223    raw_stdout: &'a str,
224    raw_stderr: &'a str,
225    started_at: &'a chrono::DateTime<Utc>,
226    error_override: Option<String>,
227    elapsed_secs: Option<u64>,
228    push_failed: bool,
229}
230
231fn parse_success_output(
232    task_id: &str,
233    agent_name: &str,
234    agent_runner: &dyn agents::AgentRunner,
235    raw_stdout: &str,
236) -> Result<agents::ParsedResponse, agents::AgentError> {
237    // The last result envelope is the agent's final answer and carries the run's tokens and cost.
238    // Claude writes terminal_reason:completed on every run, so this goes before that shortcut
239    let agent_result = agents::find_agent_result(agent_name, raw_stdout);
240    if let Some(result) = &agent_result
241        && !result.is_error
242        && let Ok(response) = parser::parse(&result.result_text)
243    {
244        return Ok(agents::ParsedResponse {
245            response,
246            input_tokens: result.input_tokens,
247            output_tokens: result.output_tokens,
248            cost_usd: result.cost_usd,
249            duration_ms: result.duration_ms,
250        });
251    }
252
253    // Quick NDJSON terminal_reason:completed shortcut: when the agent
254    // emitted a session envelope that explicitly signals completion we
255    // prefer to interpret that as a successful run (or at least a
256    // soft/empty success) rather than immediately classifying an error
257    // from the trailing process output.  Do not override explicit
258    // in-envelope `is_error: true` signals.
259    if raw_stdout.contains("\"terminal_reason\":\"completed\"")
260        && !raw_stdout.contains("\"is_error\":true")
261    {
262        // Try to recover a structured response from assistant messages first
263        let assistant_text = agents::collect_assistant_messages_text(agent_name, raw_stdout);
264        if !assistant_text.is_empty() {
265            if let Ok(response) = parser::parse(&assistant_text) {
266                tracing::debug!(
267                    task_id,
268                    agent = agent_name,
269                    "found AgentResponse JSON in earlier assistant message (terminal_reason:completed)"
270                );
271                return Ok(agents::ParsedResponse {
272                    response,
273                    input_tokens: None,
274                    output_tokens: None,
275                    cost_usd: None,
276                    duration_ms: None,
277                });
278            }
279            if let Some(response) = agents::synthesize_response_from_text(&assistant_text) {
280                tracing::warn!(
281                    task_id,
282                    agent = agent_name,
283                    "synthesizing response from assistant message due to terminal_reason:completed"
284                );
285                return Ok(agents::ParsedResponse {
286                    response,
287                    input_tokens: None,
288                    output_tokens: None,
289                    cost_usd: None,
290                    duration_ms: None,
291                });
292            }
293        }
294        // If we couldn't recover structured data from earlier assistant
295        // messages, fall through and let the regular NDJSON envelope parsing
296        // / agent-specific parsers handle the terminal_reason case. This
297        // preserves the existing behavior where an empty `result` field
298        // combined with exit != 0 becomes an InvalidResponse that is then
299        // interpreted by parse_session_output (see tests).
300    }
301
302    // 1. Try to use the unified agent result extractor first
303    if let Some(agent_result) = agent_result {
304        // Treat is_error as authoritative for the success path
305        if agent_result.is_error {
306            // Classify the error from the agent's result text rather than
307            // blindly returning AgentFailed — this preserves rate-limit, auth,
308            // and other actionable error types for proper cooldown/reroute.
309            // Without this, e.g. a claude session-limit error with is_error=true
310            // would be downgraded to AgentFailed, losing the RateLimit signal
311            // that the fallback chain needs for UsageLimit recovery (issue #3292).
312            if let Some(e) = agents::patterns::detect_rate_limit(&agent_result.result_text) {
313                return Err(e);
314            }
315            if let Some(e) = agents::patterns::detect_auth_error(&agent_result.result_text) {
316                return Err(e);
317            }
318            if let Some(e) = agents::patterns::detect_model_unavailable(&agent_result.result_text) {
319                return Err(e);
320            }
321            return Err(agents::AgentError::AgentFailed {
322                message: agent_result.result_text,
323            });
324        }
325
326        // Strategy 2: scan all earlier assistant messages in the NDJSON
327        // stream for valid AgentResponse JSON. Agents often emit the
328        // JSON in an intermediate turn and then summarise in prose as
329        // their final output, so the `type:result` text may be pure
330        // narrative while a prior assistant turn contains the JSON.
331        let assistant_text = agents::collect_assistant_messages_text(agent_name, raw_stdout);
332        if !assistant_text.is_empty()
333            && let Ok(response) = parser::parse(&assistant_text)
334        {
335            tracing::debug!(
336                task_id,
337                agent = agent_name,
338                "found AgentResponse JSON in earlier assistant message"
339            );
340            return Ok(agents::ParsedResponse {
341                response,
342                input_tokens: agent_result.input_tokens,
343                output_tokens: agent_result.output_tokens,
344                cost_usd: agent_result.cost_usd,
345                duration_ms: agent_result.duration_ms,
346            });
347        }
348
349        // Only fall back to text synthesis when parsing fails
350        if let Some(response) = agents::synthesize_response_from_text(&agent_result.result_text) {
351            tracing::warn!(
352                task_id,
353                agent = agent_name,
354                "parse failed on agent result, synthesizing response from plain text"
355            );
356            return Ok(agents::ParsedResponse {
357                response,
358                input_tokens: agent_result.input_tokens,
359                output_tokens: agent_result.output_tokens,
360                cost_usd: agent_result.cost_usd,
361                duration_ms: agent_result.duration_ms,
362            });
363        }
364
365        // NDJSON envelope fallback: the envelope explicitly reports
366        // success (is_error=false) but the result content is domain
367        // JSON or plain prose that doesn't match AgentResponse schema
368        // and doesn't contain completion keywords.  Rather than
369        // returning InvalidResponse (which causes the task to re-run),
370        // synthesize a minimal "done" response from the result text.
371        //
372        // Guard against kimi/minimax edge case: when terminal_reason=completed
373        // but result text is empty, synthesize returns None, parse_success_output
374        // returns InvalidResponse, and parse_session_output can intercept it
375        // before classify_error. Requiring non-empty result here means the
376        // envelope fallback only fires when we have actual text to report.
377        if !agent_result.is_error
378            && result_envelope_completed(raw_stdout)
379            && !agent_result.result_text.is_empty()
380        {
381            let summary_end = agents::truncate_at_char_boundary(&agent_result.result_text, 500);
382            let summary = agent_result.result_text[..summary_end].to_string();
383            tracing::warn!(
384                task_id,
385                agent = agent_name,
386                "NDJSON envelope indicates success but result lacks AgentResponse \
387                 schema — synthesizing done response"
388            );
389            return Ok(agents::ParsedResponse {
390                response: AgentResponse {
391                    status: "done".to_string(),
392                    summary,
393                    ..Default::default()
394                },
395                input_tokens: agent_result.input_tokens,
396                output_tokens: agent_result.output_tokens,
397                cost_usd: agent_result.cost_usd,
398                duration_ms: agent_result.duration_ms,
399            });
400        }
401    }
402
403    // 2. Fall back to generic agent-specific parser if extraction failed
404    match agent_runner.parse_response(raw_stdout) {
405        Ok(parsed) => Ok(parsed),
406        Err(agents::AgentError::InvalidResponse { raw }) => {
407            if let Some(response) = agents::synthesize_response_from_text(&raw) {
408                tracing::warn!(
409                    task_id,
410                    agent = agent_name,
411                    "parse failed, synthesizing response from plain text"
412                );
413                // Try to extract token metadata from the original raw NDJSON
414                // output (not the truncated/extracted text from the error), so
415                // cost tracking isn't lost for agents that return free-form text.
416                let agent_result = agents::find_agent_result(agent_name, raw_stdout);
417                let input_tokens = agent_result.as_ref().and_then(|r| r.input_tokens);
418                let output_tokens = agent_result.as_ref().and_then(|r| r.output_tokens);
419                let cost_usd = agent_result.as_ref().and_then(|r| r.cost_usd);
420                let duration_ms = agent_result.as_ref().and_then(|r| r.duration_ms);
421                Ok(agents::ParsedResponse {
422                    response,
423                    input_tokens,
424                    output_tokens,
425                    cost_usd,
426                    duration_ms,
427                })
428            } else {
429                Err(agents::AgentError::InvalidResponse { raw })
430            }
431        }
432        Err(err) => Err(err),
433    }
434}
435
436/// Whether the last `type: result` line reports `subtype: success` or
437/// `terminal_reason: completed`. Grok omits `terminal_reason`.
438fn result_envelope_line(raw_stdout: &str) -> Option<serde_json::Value> {
439    raw_stdout
440        .lines()
441        .rev()
442        .filter_map(|l| serde_json::from_str::<serde_json::Value>(l.trim()).ok())
443        .find(|v| v.get("type").and_then(|t| t.as_str()) == Some("result"))
444}
445
446fn has_result_envelope(raw_stdout: &str) -> bool {
447    result_envelope_line(raw_stdout).is_some()
448}
449
450fn result_envelope_completed(raw_stdout: &str) -> bool {
451    result_envelope_line(raw_stdout).is_some_and(|v| {
452        v.get("subtype").and_then(|t| t.as_str()) == Some("success")
453            || v.get("terminal_reason").and_then(|t| t.as_str()) == Some("completed")
454    })
455}
456
457// A timeout kill has no result envelope. Salvage only the wrap-up shape: a successful
458// `git commit`, then only shell calls that all returned ok. Returns the last assistant text
459fn salvage_wrapup(raw: &str) -> Option<String> {
460    // (tool name, command, finished ok) in call order
461    let mut calls: Vec<(String, String, Option<bool>)> = Vec::new();
462    let mut last_text = String::new();
463    for v in raw
464        .lines()
465        .filter_map(|l| serde_json::from_str::<Value>(l.trim()).ok())
466    {
467        let blocks = v.pointer("/message/content").and_then(|c| c.as_array());
468        for b in blocks.into_iter().flatten() {
469            match b.get("type").and_then(|t| t.as_str()) {
470                Some("tool_use") => calls.push((
471                    b["name"].as_str().unwrap_or("").to_string(),
472                    b.pointer("/input/command")
473                        .and_then(|c| c.as_str())
474                        .unwrap_or("")
475                        .to_string(),
476                    None,
477                )),
478                Some("text") if v["type"] == "assistant" => {
479                    last_text = b["text"].as_str().unwrap_or("").to_string();
480                }
481                Some("tool_result") => {
482                    let ok = b.get("is_error").and_then(serde_json::Value::as_bool) != Some(true);
483                    // results arrive in call order, fill the first unanswered call
484                    if let Some(c) = calls.iter_mut().find(|c| c.2.is_none()) {
485                        c.2 = Some(ok);
486                    }
487                }
488                _ => {}
489            }
490        }
491    }
492    let commit = calls
493        .iter()
494        .rposition(|(_, cmd, ok)| cmd.contains("git commit") && *ok == Some(true))?;
495    let tail_ok = calls[commit..].iter().all(|(name, _, ok)| {
496        matches!(name.as_str(), "Bash" | "run_terminal_cmd") && *ok == Some(true)
497    });
498    let lower = last_text.to_lowercase();
499    if !tail_ok || lower.contains("still running") || lower.contains("in progress") {
500        return None;
501    }
502    Some(last_text.chars().take(500).collect())
503}
504
505/**
506 * Tries `parse_success_output` first when stdout is non-empty. If that succeeds,
507 * returns success regardless of exit code. If it fails and `exit_code != 0`,
508 * checks for `"terminal_reason":"completed"` or cost-telemetry-only stdout
509 * (`"costUSD":`) before falling back to `classify_error`, so valid sessions that
510 * exit 1 produce `InvalidResponse` (soft error, re-route) instead of a garbled
511 * `Unknown`/`Auth` error (hard failure with spurious cooldown)
512 **/
513pub fn parse_session_output(
514    task_id: &str,
515    agent_name: &str,
516    agent_runner: &dyn agents::AgentRunner,
517    session_output: &session::SessionOutput,
518) -> Result<agents::ParsedResponse, agents::AgentError> {
519    // Some agents may emit NDJSON/telemetry to stderr instead of stdout. Combine
520    // both streams so parse_success_output can inspect either place for the
521    // terminal_reason/cost telemetry envelope. Only skip parsing when both are
522    // empty so we still attempt recovery when stderr contains the envelope.
523    let combined_output = format!("{}{}", session_output.raw_stdout, session_output.raw_stderr);
524    if !(session_output.raw_stdout.is_empty() && session_output.raw_stderr.is_empty()) {
525        match parse_success_output(task_id, agent_name, agent_runner, &combined_output) {
526            Ok(parsed) => return Ok(parsed),
527            Err(err) => {
528                if session_output.exit_code == 124
529                    && let Some(summary) = salvage_wrapup(&combined_output)
530                {
531                    tracing::warn!(
532                        task_id,
533                        agent = agent_name,
534                        "timed out after commit during wrap-up, salvaging as done"
535                    );
536                    return Ok(agents::ParsedResponse {
537                        response: AgentResponse {
538                            status: "done".to_string(),
539                            summary,
540                            ..Default::default()
541                        },
542                        input_tokens: None,
543                        output_tokens: None,
544                        cost_usd: None,
545                        duration_ms: None,
546                    });
547                }
548                // If parse_success_output already returned a classified error
549                // (RateLimit, AgentFailed, Timeout, Auth, etc.), propagate it
550                // directly — only InvalidResponse (unparseable output) should
551                // be intercepted by the terminal_reason:completed / cost-telemetry
552                // guard below. Without this, e.g. a claude session-limit error
553                // with terminal_reason=completed would be downgraded from RateLimit
554                // to InvalidResponse → Failed, losing the correct cooldown/reroute.
555                if !matches!(err, agents::AgentError::InvalidResponse { .. }) {
556                    return Err(err);
557                }
558                if session_output.exit_code != 0 {
559                    // kimi/minimax/glm sometimes emit valid NDJSON with
560                    // "terminal_reason":"completed" or cost-only telemetry
561                    // ("costUSD":) but exit 1 due to a post-session hook or
562                    // minor CLI issue. Scanning the tail would just grab JSON
563                    // metadata, producing a garbled error. Detect either
564                    // completion signal and treat as a soft parse error so the
565                    // task re-routes instead of hitting a false cooldown from
566                    // Unknown or Auth error variants. Use the combined output
567                    // (stdout+stderr) so envelopes written to stderr are
568                    // observed.
569                    let combined = &combined_output;
570                    let is_completed_signal =
571                        combined.contains("\"terminal_reason\":\"completed\"");
572                    // GLM emits a cost-telemetry JSON object (containing
573                    // "costUSD":) as its final stdout/stderr line on clean
574                    // exit even when the process exits with code 1. If the
575                    // only content is such telemetry (no is_error=true in the
576                    // envelope), treat it the same as terminal_reason:completed.
577                    let is_cost_telemetry_only = combined.contains("\"costUSD\":")
578                        && !combined.contains("\"is_error\":true");
579                    if is_completed_signal || is_cost_telemetry_only {
580                        let raw_tail = agents::patterns::safe_tail(combined, 300);
581                        return Err(agents::AgentError::InvalidResponse {
582                            raw: raw_tail.to_string(),
583                        });
584                    }
585                    return Err(agent_runner.classify_error_with_elapsed(
586                        session_output.exit_code,
587                        &session_output.raw_stdout,
588                        &session_output.raw_stderr,
589                        session_output.elapsed_secs,
590                    ));
591                }
592                if !has_result_envelope(&combined_output) {
593                    return Err(agents::AgentError::Interrupted {
594                        message: "exited without a result envelope".to_string(),
595                    });
596                }
597                return Err(err);
598            }
599        }
600    }
601
602    if session_output.exit_code != 0 {
603        return Err(agent_runner.classify_error_with_elapsed(
604            session_output.exit_code,
605            &session_output.raw_stdout,
606            &session_output.raw_stderr,
607            session_output.elapsed_secs,
608        ));
609    }
610
611    Err(agents::AgentError::Unknown {
612        exit_code: session_output.exit_code,
613        message: "empty-output-exit0: opencode returned exit 0 with empty stdout".to_string(),
614    })
615}
616
617/// The repo has no project directory: not in the active config, no legacy
618/// `project_dir`, no bare clone. It fails before any agent starts
619#[derive(Debug)]
620pub(crate) struct NoProjectDir {
621    pub(crate) repo: String,
622    pub(crate) bare_clone: PathBuf,
623}
624
625impl NoProjectDir {
626    /// Block reason for a task of this repo
627    pub(crate) const BLOCK_REASON: &str = "project is not configured and has no bare clone";
628}
629
630impl fmt::Display for NoProjectDir {
631    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
632        write!(
633            f,
634            "no resolvable project directory for repo {} — not in active config, no bare clone at {}, no legacy project_dir set",
635            self.repo,
636            self.bare_clone.display()
637        )
638    }
639}
640
641impl error::Error for NoProjectDir {}
642
643/// Task runner configuration.
644pub struct TaskRunner {
645    /// Repository slug (owner/repo)
646    repo: String,
647    /// Path to the orch home directory
648    orch_home: PathBuf,
649    /// Unified task store for run audit trail
650    store: Option<Arc<TaskStore>>,
651}
652
653impl Debug for TaskRunner {
654    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
655        f.debug_struct("TaskRunner")
656            .field("repo", &self.repo)
657            .field("orch_home", &self.orch_home)
658            .finish_non_exhaustive()
659    }
660}
661
662impl TaskRunner {
663    pub fn new(repo: String) -> Self {
664        let orch_home = home::orch_home().unwrap_or_else(|e| {
665            tracing::warn!(error = %e, "orch_home() failed — falling back to /tmp/.orch");
666            PathBuf::from("/tmp").join(".orch")
667        });
668
669        Self {
670            repo,
671            orch_home,
672            store: None,
673        }
674    }
675
676    /// Set the unified task store for run audit trail.
677    #[must_use]
678    pub fn with_store(mut self, store: Arc<TaskStore>) -> Self {
679        self.store = Some(store);
680        self
681    }
682
683    /// Read a field from the task store.
684    async fn get_field(&self, task_id: &str, field: &str) -> Option<String> {
685        let task = store::opt_store_get_task(&self.store, &self.repo, task_id).await?;
686        match field {
687            "route_reason" => Some(task.route_reason),
688            "worktree" => Some(task.worktree),
689            "attempts" => Some(task.attempts.to_string()),
690            "delegations" => serde_json::to_string(&task.delegations).ok(),
691            "summary" => Some(task.summary),
692            "last_error" => Some(task.last_error),
693            "agent" => task.agent,
694            _ => None,
695        }
696    }
697
698    /// Keep a stored fallback on reroute, wipe agent/model when the run's agent is still stored
699    pub(crate) async fn clear_agent_on_reroute_unless_fallback(
700        &self,
701        task_id: &str,
702        ran_agent: &str,
703    ) {
704        let stored = store::opt_store_get_task(&self.store, &self.repo, task_id).await;
705        if Self::reroute_keeps_stored_agent(stored.as_ref(), ran_agent) {
706            tracing::debug!(
707                task_id,
708                "stored fallback agent present, skipping agent/model clear"
709            );
710            return;
711        }
712        if let Err(e) = store::store_set_result(
713            &self.store,
714            &self.repo,
715            task_id,
716            &[
717                ("agent", serde_json::json!("")),
718                ("model", serde_json::json!("")),
719            ],
720        )
721        .await
722        {
723            tracing::warn!(task_id, err = %e, "failed to clear agent/model for reroute in store");
724        }
725    }
726
727    /// True when a fallback is already stored and must survive this reroute
728    fn reroute_keeps_stored_agent(stored: Option<&store::Task>, ran_agent: &str) -> bool {
729        let Some(task) = stored else {
730            return false;
731        };
732        if !matches!(task.status, TaskStatus::InProgress) {
733            return true;
734        }
735        let stored_agent = task.agent.as_deref().unwrap_or("");
736        !stored_agent.is_empty() && stored_agent != ran_agent
737    }
738
739    async fn build_run_audit(&self, input: RunAuditInput<'_>) -> RunAudit {
740        // Filter empty strings so that a blank last_error does not shadow the
741        // actual parse_result error.  Without this, `Option::or(Some(""))` would
742        // return `Some("")` and prevent the `unwrap_or_else` fallback from ever
743        // running, causing task_runs.error to be written as "" even when the
744        // agent returned a classifiable error (fixes issue #2527).
745        let last_error = self
746            .get_field(input.task_id, "last_error")
747            .await
748            .filter(|s| !s.is_empty());
749        let outcome =
750            classify_run_outcome(input.status, input.parse_result, input.push_failed).to_string();
751
752        let error = if outcome == "success" {
753            String::new()
754        } else {
755            // Prefer this run's parsed error over a last_error left by a previous
756            // agent's failover. last_error stays the Err/EarlyReturn fallback (#3851)
757            let error = input
758                .error_override
759                .as_deref()
760                .filter(|s| !s.is_empty())
761                .map(str::to_string)
762                .or_else(|| match input.parse_result {
763                    Ok(parsed) => parsed
764                        .response
765                        .error
766                        .as_deref()
767                        .filter(|s| !s.is_empty())
768                        .map(str::to_string),
769                    Err(_) => None,
770                })
771                .or(last_error)
772                .unwrap_or_else(|| match input.parse_result {
773                    Ok(_) => String::new(),
774                    Err(err) => err.to_string(),
775                });
776
777            if error.is_empty() {
778                if input.status == "blocked" {
779                    match input.parse_result {
780                        Ok(parsed) if !parsed.response.summary.is_empty() => {
781                            parsed.response.summary.clone()
782                        }
783                        _ => "task blocked without explicit error".to_string(),
784                    }
785                } else if matches!(
786                    input.status,
787                    "done"
788                        | "completed"
789                        | "in_progress"
790                        | "in_review"
791                        | "needs_review"
792                        | "routed"
793                        | "new"
794                ) {
795                    // Canonical non-success statuses have no error message (e.g. "routed").
796                    String::new()
797                } else {
798                    // Non-canonical status with no error message — surface the status
799                    // explicitly so operators can identify which statuses need to be
800                    // added to the normalization map in parser.rs.
801                    format!("unrecognized status: {}", input.status)
802                }
803            } else {
804                error
805            }
806        };
807
808        let total_cost_usd = input
809            .parse_result
810            .as_ref()
811            .ok()
812            .and_then(|p| p.cost_usd)
813            .unwrap_or(0.0);
814
815        let (input_tokens, output_tokens) = extract_run_tokens(input.parse_result);
816
817        #[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
818        let duration_secs = input
819            .elapsed_secs
820            .map_or_else(|| run_duration_secs(input.started_at), |s| s as f64);
821
822        RunAudit {
823            stdout: input.raw_stdout.to_string(),
824            stderr: input.raw_stderr.to_string(),
825            parsed_response: serialize_parsed_response(input.parse_result, input.raw_stdout),
826            outcome,
827            error,
828            input_tokens,
829            output_tokens,
830            total_cost_usd,
831            duration_secs,
832        }
833    }
834
835    /// The blocked outcome for a sandbox that cannot start. No agent ran, so none is cooled down
836    async fn sandbox_blocked(&self, task_id: &str, reason: String) -> RunExecution {
837        tracing::error!(task_id, error = %reason, "blocking task");
838        if let Err(e) = store::store_set_result(
839            &self.store,
840            &self.repo,
841            task_id,
842            &[
843                ("last_error", serde_json::json!(reason)),
844                ("block_reason", serde_json::json!(reason)),
845            ],
846        )
847        .await
848        {
849            tracing::warn!(task_id, err = %e, "failed to write the sandbox block_reason");
850        }
851        RunExecution {
852            status: "blocked".to_string(),
853            exit_code: None,
854            audit: RunAudit {
855                stdout: String::new(),
856                stderr: String::new(),
857                parsed_response: String::new(),
858                outcome: "failed".to_string(),
859                error: reason,
860                input_tokens: 0,
861                output_tokens: 0,
862                total_cost_usd: 0.0,
863                duration_secs: 0.0,
864            },
865        }
866    }
867
868    /// Run a task through the full execution pipeline.
869    ///
870    /// Returns `Ok(None)` if the runner guard skipped the task (caller should not
871    /// post stale results). Returns `Ok(Some(status))` with the final task status.
872    pub async fn run(
873        &self,
874        task_id: &str,
875        agent: Option<&str>,
876        model: Option<&str>,
877        backend: Option<&dyn ExternalBackend>,
878        started_at: &chrono::DateTime<Utc>,
879    ) -> anyhow::Result<Option<RunExecution>> {
880        tracing::info!(
881            task_id,
882            agent = agent.unwrap_or("default"),
883            model = model.unwrap_or("default"),
884            "starting task execution"
885        );
886
887        // Check task guards; returns outcome indicating whether to proceed.
888        let attempts = match task_init::check_guards(task_id, &self.repo, &self.store).await {
889            Ok(task_init::GuardOutcome::Proceed(a)) => a,
890            Ok(task_init::GuardOutcome::Skip) => {
891                return Ok(None);
892            }
893            Ok(task_init::GuardOutcome::MaxAttempts) => {
894                // The caller marks the task blocked from the returned status, a task that ran out
895                // of attempts did not finish, so it must never reach done
896                if backend.is_some() {
897                    // If this was label-forced to a specific agent, remove the agent label
898                    // so that /retry can route to a different agent instead of looping.
899                    let route_reason = self
900                        .get_field(task_id, "route_reason")
901                        .await
902                        .unwrap_or_default();
903                    if route_reason.starts_with("label agent:") {
904                        let agent_label = route_reason.trim_start_matches("label ");
905                        let gh = GhHttp::new()?;
906                        match gh.remove_label(&self.repo, task_id, agent_label).await {
907                            Err(e) => {
908                                tracing::warn!(task_id, label = agent_label, err = %e, "failed to remove agent label after max attempts");
909                            }
910                            _ => {
911                                tracing::info!(
912                                    task_id,
913                                    label = agent_label,
914                                    "removed forced agent label after max attempts — /retry will use free routing"
915                                );
916                            }
917                        }
918                    }
919                }
920                return Ok(Some(RunExecution {
921                    status: "blocked".to_string(),
922                    exit_code: None,
923                    audit: RunAudit {
924                        stdout: String::new(),
925                        stderr: String::new(),
926                        parsed_response: String::new(),
927                        outcome: "failed".to_string(),
928                        error: "max attempts reached".to_string(),
929                        input_tokens: 0,
930                        output_tokens: 0,
931                        total_cost_usd: 0.0,
932                        duration_secs: 0.0,
933                    },
934                }));
935            }
936            Err(e) => return Err(e),
937        };
938
939        // Resolve project directory
940        let project_dir = self.resolve_project_dir().await?;
941
942        // Prepare task: worktree, context, prompts, and agent invocation
943        let init = task_init::prepare_task(
944            task_id,
945            agent,
946            model,
947            backend,
948            &self.repo,
949            &project_dir,
950            attempts,
951            &self.store,
952        )
953        .await?;
954
955        // A sandbox that cannot start blocks the task, which never runs without it. It is the
956        // operator's setup that fails, so no agent or model is cooled down
957        if init.invocation.sandbox
958            && let Err(e) = sandbox::argv(
959                &init.invocation,
960                &env::temp_dir().join("orch-sandbox-preflight"),
961            )
962            .await
963        {
964            return Ok(Some(self.sandbox_blocked(task_id, e.to_string()).await));
965        }
966
967        // Run agent session: spawn in tmux, wait for completion, collect output
968        let (tmux, tmux_session, session_output) = session::run_agent_session(
969            task_id,
970            &init.invocation,
971            &init.attempt_dir,
972            &self.orch_home,
973        )
974        .await;
975
976        // The pre-flight policy passed but the one built at spawn did not: block the same way
977        if let Some(reason) = session_output.sandbox_error.clone() {
978            return Ok(Some(self.sandbox_blocked(task_id, reason).await));
979        }
980
981        // Touch updated_at immediately after session exit so the stuck-task recovery
982        // timer resets before response_handler post-processing (git push, PR creation,
983        // status update) begins.  Without this, a session that ran longer than
984        // no_session_stuck_timeout (default 10 min) would be re-dispatched by the tick
985        // while the runner is still writing results.
986        store::store_touch_updated_at(&self.store, &self.repo, task_id).await;
987
988        // If silence detection (tick phase1b) already rerouted this task while the
989        // session was running, skip all fallback/review processing. Without this check the
990        // runner would call `fallback::handle_error` on the empty output, exhaust fallback
991        // agents, and mark the task `needs_review` — overwriting the rerouted status and
992        // triggering a spurious review cycle.
993        // Silence detection now sets Routed (with fallback agent) or NeedsReview (no fallback).
994        // Check for both, plus New for backward compatibility.
995        if let Some(stored) = store::opt_store_get_task(&self.store, &self.repo, task_id).await {
996            let silence_reset = matches!(
997                stored.status,
998                TaskStatus::New | TaskStatus::Routed | TaskStatus::NeedsReview
999            );
1000            if silence_reset {
1001                let status_str = stored.status.as_str();
1002                tracing::info!(
1003                    task_id,
1004                    status = status_str,
1005                    "task already reset by the engine — skipping fallback processing"
1006                );
1007                session::cleanup_session(task_id, &tmux, &tmux_session).await;
1008                let silence_err = agents::patterns::classify_from_text(0, "silence detected");
1009                let silence_parse: Result<agents::ParsedResponse, agents::AgentError> =
1010                    Err(silence_err);
1011                let audit = self
1012                    .build_run_audit(RunAuditInput {
1013                        task_id,
1014                        status: status_str,
1015                        parse_result: &silence_parse,
1016                        raw_stdout: &session_output.raw_stdout,
1017                        raw_stderr: &session_output.raw_stderr,
1018                        started_at,
1019                        error_override: Some(format!("task was reset by the engine while the session ran (status {status_str})")),
1020                        elapsed_secs: session_output.elapsed_secs,
1021                        push_failed: false,
1022                    })
1023                    .await;
1024                return Ok(Some(RunExecution {
1025                    status: status_str.to_string(),
1026                    exit_code: Some(session_output.exit_code),
1027                    audit,
1028                }));
1029            }
1030        }
1031
1032        // Log raw output for debugging agent failures
1033        let stdout_len = session_output.raw_stdout.len();
1034        let stderr_len = session_output.raw_stderr.len();
1035        let stdout_tail: String = session_output
1036            .raw_stdout
1037            .chars()
1038            .rev()
1039            .take(500)
1040            .collect::<String>()
1041            .chars()
1042            .rev()
1043            .collect();
1044        let stderr_tail: String = session_output
1045            .raw_stderr
1046            .chars()
1047            .rev()
1048            .take(500)
1049            .collect::<String>()
1050            .chars()
1051            .rev()
1052            .collect();
1053        tracing::info!(
1054            task_id,
1055            exit_code = session_output.exit_code,
1056            stdout_len,
1057            stderr_len,
1058            stdout_tail = %stdout_tail,
1059            stderr_tail = %stderr_tail,
1060            "agent raw output"
1061        );
1062
1063        // Use agent-specific parsing where possible. Previously we only attempted
1064        // parse_success_output when exit_code == 0 which caused runs that emitted
1065        // valid NDJSON but exited non-zero (kimi / minimax) to be classed as
1066        // failures. Try parsing success output first when stdout is present; if
1067        // parsing succeeds treat as success regardless of exit code. Only fall
1068        // back to error classification when parsing fails and the process exit
1069        // code indicates an error.
1070        for window in agents::claude::rate_limit_windows(&session_output.raw_stdout) {
1071            cooldown::record_rate_limit_window(&init.agent_name, &window).await;
1072        }
1073        let agent_runner = agents::get_runner(&init.agent_name);
1074        let parse_result =
1075            parse_session_output(task_id, &init.agent_name, &*agent_runner, &session_output);
1076
1077        // Write structured result.json for deterministic testing and debugging
1078        response_handler::write_result_json(
1079            &init.attempt_dir,
1080            task_id,
1081            &init.agent_name,
1082            init.model_name.as_deref(),
1083            session_output.exit_code,
1084            init.new_attempts,
1085            &parse_result,
1086            &session_output.raw_stdout,
1087            &session_output.raw_stderr,
1088        )
1089        .await;
1090
1091        // Handle outcome: success or error recovery
1092        // `fallback_error` carries the attributed error message from `handle_error()` when
1093        // the Continue path is taken.  It is passed as `error_override` to `build_run_audit`
1094        // so that stale `last_error` values from a previous agent's run are never written
1095        // into the current run's audit record (fixes misattribution bug: task_run shows
1096        // agent=kimi but error="minimax timed out...").
1097        let (final_status, push_failed, fallback_error) = match parse_result {
1098            Ok(ref parsed) => {
1099                let (status, push_failed) = response_handler::handle_success(
1100                    task_id,
1101                    parsed.clone(),
1102                    &init.wt,
1103                    &init.task_title,
1104                    &init.agent_name,
1105                    init.model_name.as_deref(),
1106                    init.new_attempts,
1107                    &self.repo,
1108                    &self.store,
1109                    &session_output.raw_stdout,
1110                )
1111                .await?;
1112                (status, push_failed, None)
1113            }
1114            Err(ref agent_err) => {
1115                match fallback::handle_error(
1116                    task_id,
1117                    agent_err,
1118                    &init.agent_name,
1119                    &*agent_runner,
1120                    init.model_name.as_deref(),
1121                    init.complexity.as_deref(),
1122                    init.new_attempts,
1123                    &self.store,
1124                    &self.repo,
1125                )
1126                .await?
1127                {
1128                    fallback::ErrorHandleResult::EarlyReturn { status } => {
1129                        // Task rerouted — clean up tmux session and env secrets
1130                        session::cleanup_session(task_id, &tmux, &tmux_session).await;
1131                        // The fallback handler already writes a sanitized `last_error`
1132                        // into the store for EarlyReturn paths (reroutes / needs_review).
1133                        // Passing the raw agent_err here would record noisy provider
1134                        // payloads (e.g. full api_retry NDJSON). Leave error_override
1135                        // as None so build_run_audit will prefer the store's last_error
1136                        // (see src/engine/runner/fallback.rs which persists a compact
1137                        // summary for rate-limit and reroute cases).
1138                        let audit = self
1139                            .build_run_audit(RunAuditInput {
1140                                task_id,
1141                                status: &status,
1142                                parse_result: &parse_result,
1143                                raw_stdout: &session_output.raw_stdout,
1144                                raw_stderr: &session_output.raw_stderr,
1145                                started_at,
1146                                error_override: None,
1147                                elapsed_secs: session_output.elapsed_secs,
1148                                push_failed: false,
1149                            })
1150                            .await;
1151                        return Ok(Some(RunExecution {
1152                            status,
1153                            exit_code: Some(session_output.exit_code),
1154                            audit,
1155                        }));
1156                    }
1157                    fallback::ErrorHandleResult::Continue { status, error } => {
1158                        (status, false, Some(error))
1159                    }
1160                }
1161            }
1162        };
1163
1164        // Kill tmux session if still alive
1165        session::cleanup_session(task_id, &tmux, &tmux_session).await;
1166
1167        // For no-op done tasks (agent finished with no code changes and no PR),
1168        // clean up the worktree and branch immediately — tmux session is already
1169        // dead so the tmux guard won't block removal.
1170        if final_status.as_str() == "done"
1171            && let Some(ref st) = self.store
1172            && let Err(e) = cleanup::cleanup_task_worktree(task_id, &self.repo, st).await
1173        {
1174            tracing::warn!(task_id, error = ?e, "worktree cleanup failed after no-op done");
1175        }
1176
1177        // Return final status and the session exit code so callers can record it.
1178        // Use `fallback_error` as error_override when available: it carries the error message
1179        // attributed to the current agent (set by handle_error), preventing stale last_error
1180        // values from a prior run from contaminating this run's audit record.
1181        let audit = self
1182            .build_run_audit(RunAuditInput {
1183                task_id,
1184                status: &final_status,
1185                parse_result: &parse_result,
1186                raw_stdout: &session_output.raw_stdout,
1187                raw_stderr: &session_output.raw_stderr,
1188                started_at,
1189                error_override: fallback_error,
1190                elapsed_secs: session_output.elapsed_secs,
1191                push_failed,
1192            })
1193            .await;
1194
1195        Ok(Some(RunExecution {
1196            status: final_status.clone(),
1197            exit_code: Some(session_output.exit_code),
1198            audit,
1199        }))
1200    }
1201
1202    /// Record task execution metrics to the database.
1203    #[expect(
1204        clippy::too_many_arguments,
1205        reason = "a metric row needs the agent, model, route, timing and outcome together"
1206    )]
1207    async fn record_metrics(
1208        &self,
1209        task_id: &str,
1210        agent_name: &str,
1211        model_name: Option<&str>,
1212        route_result: Option<&RouteResult>,
1213        started_at: &chrono::DateTime<Utc>,
1214        attempts: u32,
1215        final_status: &str,
1216        error_type: Option<&str>,
1217    ) {
1218        let completed_at = Utc::now();
1219        #[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
1220        let duration_seconds = (completed_at - *started_at).num_milliseconds() as f64 / 1000.0;
1221
1222        // Classify outcome based on the final status and optional error_type/last_error
1223        let outcome = match final_status {
1224            "done" | "in_progress" | "in_review" => "success",
1225            "new" => "rerouted",
1226            "needs_review" => classify_run_error_type(error_type.unwrap_or("")),
1227            _ => "unknown",
1228        };
1229
1230        let complexity = route_result.map(|r| r.complexity.clone());
1231        let files_changed = git_ops::count_changed_files(&PathBuf::from(
1232            self.get_field(task_id, "worktree")
1233                .await
1234                .unwrap_or_default(),
1235        ))
1236        .await
1237        .unwrap_or(0);
1238
1239        if let Some(ref store) = self.store {
1240            // Only set error_type for non-success outcomes
1241            let db_error_type: Option<String> = if outcome == "success" {
1242                None
1243            } else {
1244                Some(outcome.to_string())
1245            };
1246
1247            let usage = store::get_token_usage(&self.store, &self.repo, task_id).await;
1248            let total_cost = store::get_reported_cost(&self.store, &self.repo, task_id).await;
1249            let input_tokens = if usage.input_tokens > 0 {
1250                Some(i64::try_from(usage.input_tokens).unwrap_or(i64::MAX))
1251            } else {
1252                None
1253            };
1254            let output_tokens = if usage.output_tokens > 0 {
1255                Some(i64::try_from(usage.output_tokens).unwrap_or(i64::MAX))
1256            } else {
1257                None
1258            };
1259
1260            let metric = InsertTaskMetric {
1261                repo: &self.repo,
1262                task_id,
1263                agent: agent_name,
1264                model: model_name,
1265                complexity: complexity.as_deref(),
1266                outcome,
1267                duration_seconds,
1268                started_at,
1269                completed_at: &completed_at,
1270                attempts: i32::try_from(attempts.saturating_add(1)).unwrap_or(i32::MAX),
1271                files_changed: i32::try_from(files_changed).unwrap_or(i32::MAX),
1272                error_type: db_error_type.as_deref(),
1273                input_tokens,
1274                output_tokens,
1275                total_cost_usd: total_cost,
1276            };
1277
1278            if let Err(e) = store.insert_task_metric(&metric).await {
1279                tracing::error!(task_id, ?e, "failed to record task metrics");
1280            }
1281        }
1282    }
1283
1284    /// Run a task with full engine context (backend, tmux, capture).
1285    ///
1286    /// Called by the engine dispatch loop with richer context.
1287    /// Returns a `WeightSignal` for the engine to feed back to the router.
1288    pub async fn run_with_context(
1289        &self,
1290        task: &ExternalTask,
1291        backend: &Arc<dyn ExternalBackend>,
1292        _tmux: &Arc<TmuxManager>,
1293        route_result: Option<&RouteResult>,
1294    ) -> anyhow::Result<WeightSignal> {
1295        let task_id = &task.id.0;
1296        let agent = route_result.map(|r| r.agent.as_str());
1297        let agent_name = agent.unwrap_or("claude").to_string();
1298        let model = route_result.and_then(|r| r.model.as_deref());
1299
1300        // Record start time for metrics (before any work begins)
1301        let started_at = Utc::now();
1302
1303        // Store task info for prompt building (title/body already in store tasks table)
1304
1305        // Record run start in task_runs audit trail
1306        let run_audit_id = if let Some(ref store) = self.store {
1307            let attempt: i32 = self
1308                .get_field(task_id, "attempts")
1309                .await
1310                .and_then(|s| s.parse().ok())
1311                .unwrap_or(0)
1312                + 1;
1313            match store.resolve_task_id(&self.repo, task_id).await {
1314                Ok(Some(store_id)) => {
1315                    match store
1316                        .start_run(&crate::store::StartRun {
1317                            task_id: store_id,
1318                            attempt,
1319                            run_type: "agent",
1320                            agent: &agent_name,
1321                            model: model.unwrap_or(""),
1322                            command: &format!(
1323                                "{} --model {}",
1324                                agent_name,
1325                                model.unwrap_or("default")
1326                            ),
1327                            prompt: &task.body,
1328                        })
1329                        .await
1330                    {
1331                        Ok(run_id) => Some(run_id),
1332                        Err(e) => {
1333                            tracing::warn!(
1334                                task_id,
1335                                error = %e,
1336                                "failed to record run start in audit trail"
1337                            );
1338                            None
1339                        }
1340                    }
1341                }
1342                _ => None,
1343            }
1344        } else {
1345            None
1346        };
1347
1348        // Run the task.
1349        // Always finalize the run audit row, even on early-return/error paths.
1350        let run_result = match self
1351            .run(task_id, agent, model, Some(&**backend), &started_at)
1352            .await
1353        {
1354            Ok(run_result) => run_result,
1355            Err(e) => {
1356                if let Some(run_id) = run_audit_id
1357                    && let Some(ref store) = self.store
1358                    && let Err(complete_err) = store
1359                        .complete_run(&crate::store::CompleteRun {
1360                            run_id,
1361                            exit_code: Some(-1),
1362                            stdout: "",
1363                            stderr: "",
1364                            parsed: "",
1365                            outcome: "failed",
1366                            error: &e.to_string(),
1367                            tokens: RunTokenUsage::default(),
1368                        })
1369                        .await
1370                {
1371                    tracing::warn!(
1372                        task_id,
1373                        run_id,
1374                        error = %complete_err,
1375                        "failed to record run completion in audit trail after runner error"
1376                    );
1377                }
1378                return Err(e);
1379            }
1380        };
1381
1382        // If the runner guard skipped the task, do not re-post stale data as a new comment.
1383        let (status, exit_code_opt, run_audit) = if let Some(result) = run_result {
1384            (result.status, result.exit_code, result.audit)
1385        } else {
1386            if let Some(run_id) = run_audit_id
1387                && let Some(ref store) = self.store
1388                && let Err(e) = store
1389                    .complete_run(&crate::store::CompleteRun {
1390                        run_id,
1391                        exit_code: Some(-1),
1392                        stdout: "",
1393                        stderr: "",
1394                        parsed: "",
1395                        outcome: "aborted",
1396                        error: "run skipped by guard",
1397                        tokens: RunTokenUsage::default(),
1398                    })
1399                    .await
1400            {
1401                tracing::warn!(
1402                    task_id,
1403                    run_id,
1404                    error = %e,
1405                    "failed to record skipped run completion in audit trail"
1406                );
1407            }
1408            tracing::info!(task_id, "guard skipped task — not posting stale result");
1409            return Ok(WeightSignal::None);
1410        };
1411
1412        // Process delegations if the agent requested subtasks
1413        let delegations_raw = self
1414            .get_field(task_id, "delegations")
1415            .await
1416            .unwrap_or_default();
1417        if !delegations_raw.is_empty() {
1418            match serde_json::from_str::<Vec<Delegation>>(&delegations_raw) {
1419                Ok(delegations) if !delegations.is_empty() => {
1420                    self.process_delegations(task, &delegations, backend)
1421                        .await?;
1422                    if let Err(e) = store::store_set_result(
1423                        &self.store,
1424                        &self.repo,
1425                        task_id,
1426                        &[("delegations", serde_json::json!([]))],
1427                    )
1428                    .await
1429                    {
1430                        tracing::warn!(task_id, err = %e, "failed to clear delegations in store");
1431                    }
1432                }
1433                Ok(_) => {} // empty list, nothing to do
1434                Err(e) => {
1435                    tracing::error!(task_id, error = %e, "corrupt delegations JSON — clearing");
1436                    if let Err(e2) = store::store_set_result(
1437                        &self.store,
1438                        &self.repo,
1439                        task_id,
1440                        &[("delegations", serde_json::json!([]))],
1441                    )
1442                    .await
1443                    {
1444                        tracing::warn!(task_id, err = %e2, "failed to clear corrupt delegations in store");
1445                    }
1446                    if let Err(e2) = store::store_set_result(
1447                        &self.store,
1448                        &self.repo,
1449                        task_id,
1450                        &[(
1451                            "last_error",
1452                            serde_json::json!(format!("delegation parse failed: {e}")),
1453                        )],
1454                    )
1455                    .await
1456                    {
1457                        tracing::warn!(task_id, err = %e2, "failed to write delegation_parse_failed last_error to store");
1458                    }
1459                }
1460            }
1461        }
1462
1463        // Post result to GitHub
1464        let summary = self.get_field(task_id, "summary").await.unwrap_or_default();
1465        let last_error = self
1466            .get_field(task_id, "last_error")
1467            .await
1468            .unwrap_or_default();
1469
1470        // Determine weight signal based on outcome.
1471        // Any reroute ("new"/"routed") should avoid review-gate and go back through
1472        // normal dispatch, regardless of why it was rerouted.
1473        let is_rerouted = status == "new" || status == "routed";
1474        if is_rerouted {
1475            self.clear_agent_on_reroute_unless_fallback(task_id, &agent_name)
1476                .await;
1477        }
1478        // Determine weight signal based on outcome.
1479        // Reroutes should only emit RateLimited when the underlying error is genuinely
1480        // a rate limit (not silence detection, timeouts, parse errors, etc.). This prevents
1481        // false cooldown cascades where non-rate-limit failures are treated as rate limits.
1482        let is_rate_limit_error = classify_run_error_type(&last_error) == "rate_limit";
1483        let weight_signal = if is_rerouted {
1484            if is_rate_limit_error {
1485                WeightSignal::RateLimited {
1486                    agent: agent_name.clone(),
1487                }
1488            } else {
1489                // Non-rate-limit reroutes (silence detection, timeout, parse error): signal
1490                // the engine to keep the task in "new" for re-routing. Using Rerouted (not
1491                // None) prevents tick from converting the silence-reset back to needs_review,
1492                // which would trigger the no-code review shortcut and falsely mark it done.
1493                WeightSignal::Rerouted
1494            }
1495        } else if status == "needs_review" {
1496            // Rate limit errors that escalate to needs_review must NOT reset backoff
1497            // counters (issue #2153).  These errors indicate the agent is genuinely
1498            // unavailable — resetting would wipe the cooldown and cause the next dispatch
1499            // to retry the same rate-limited agent immediately.
1500            //
1501            // Detection: classify_run_error_type() confirms "rate limit" in last_error.
1502            // Reuse the same classifier as `is_rate_limit_error` above rather than a raw
1503            // substring check — billing-cycle exhaustion messages (e.g. Kimi's "usage
1504            // limit for this billing cycle") also contain "usage limit"/"rate limit", so
1505            // a bare substring check would record a rate-limit event into the router's
1506            // health signal for a condition that isn't a genuine rate limit (issue #3529).
1507            let signal = needs_review_signal(&last_error, &agent_name);
1508            match &signal {
1509                WeightSignal::RateLimited { .. } => {
1510                    // Record in the metrics store so the router's health check detects it.
1511                    if let Some(ref store) = self.store
1512                        && let Err(e) = store
1513                            .record_rate_limit(&agent_name, "rate", Some(task_id))
1514                            .await
1515                    {
1516                        tracing::warn!(task_id, agent = %agent_name, err = %e, "failed to persist rate_limit event — router health check may miss this signal");
1517                    }
1518                    // Also record via the cooldown system so model_for_complexity() skips
1519                    // this model.
1520                    cooldown::record_rate_limit(&agent_name, model, &last_error).await;
1521                }
1522                WeightSignal::Success { .. } => {
1523                    // Reset backoff so the next failure starts fresh.
1524                    cooldown::record_agent_success(&agent_name, model.unwrap_or("")).await;
1525                }
1526                _ => {}
1527            }
1528            signal
1529        } else if status == "done" || status == "in_progress" || status == "in_review" {
1530            // Reset exponential backoff counters so the next failure starts fresh.
1531            cooldown::record_agent_success(&agent_name, model.unwrap_or("")).await;
1532            WeightSignal::Success {
1533                agent: agent_name.clone(),
1534            }
1535        } else if status == "blocked" {
1536            WeightSignal::Blocked
1537        } else {
1538            WeightSignal::None
1539        };
1540
1541        // Record metrics
1542        {
1543            let attempts: u32 = self
1544                .get_field(task_id, "attempts")
1545                .await
1546                .and_then(|s| s.parse().ok())
1547                .unwrap_or(0);
1548            self.record_metrics(
1549                task_id,
1550                &agent_name,
1551                model,
1552                route_result,
1553                &started_at,
1554                attempts,
1555                &status,
1556                if last_error.is_empty() {
1557                    None
1558                } else {
1559                    Some(&last_error)
1560                },
1561            )
1562            .await;
1563        }
1564
1565        // Complete run in task_runs audit trail (include exit code from runner when available)
1566        if let Some(run_id) = run_audit_id
1567            && let Some(ref store) = self.store
1568            && let Err(e) = store
1569                .complete_run(&crate::store::CompleteRun {
1570                    run_id,
1571                    exit_code: exit_code_opt,
1572                    stdout: &run_audit.stdout,
1573                    stderr: &run_audit.stderr,
1574                    parsed: &run_audit.parsed_response,
1575                    outcome: &run_audit.outcome,
1576                    error: &run_audit.error,
1577                    tokens: RunTokenUsage {
1578                        input_tokens: run_audit.input_tokens,
1579                        output_tokens: run_audit.output_tokens,
1580                        total_cost_usd: run_audit.total_cost_usd,
1581                        duration_secs: run_audit.duration_secs,
1582                    },
1583                })
1584                .await
1585        {
1586            tracing::warn!(
1587                task_id,
1588                run_id,
1589                error = %e,
1590                "failed to record run completion in audit trail"
1591            );
1592        }
1593
1594        // If task was rerouted (status=routed or new after run), update GitHub agent label
1595        // so the router doesn't re-route back to the same failed agent.
1596        if is_rerouted && !is_internal_id(task_id) {
1597            let new_agent = self.get_field(task_id, "agent").await.unwrap_or_default();
1598            if !new_agent.is_empty() && new_agent != agent_name {
1599                update_agent_label_after_reroute(
1600                    &self.repo,
1601                    task,
1602                    backend,
1603                    &agent_name,
1604                    &new_agent,
1605                )
1606                .await;
1607            }
1608        }
1609
1610        // Internal tasks: skip GitHub status/comment updates — dispatch phase handles status.
1611        if is_internal_id(task_id) {
1612            return Ok(weight_signal);
1613        }
1614
1615        // Update GitHub status
1616        let new_status = match status.as_str() {
1617            "done" => Status::Done,
1618            "in_progress" => Status::InProgress,
1619            "in_review" => Status::InReview,
1620            "blocked" => Status::Blocked,
1621            "routed" => Status::Routed, // Rerouted (agent/model already set)
1622            "new" => Status::New,       // Legacy reroute path
1623            _ => Status::NeedsReview,
1624        };
1625        // Store-first: update SQLite (must succeed), then mirror to backend.
1626        if let Some(ref store) = self.store
1627            && let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &task.id.0).await
1628        {
1629            store
1630                .update_status(store_id, tasks::status_to_task_status(new_status))
1631                .await
1632                .context("store-first status update failed in runner")?;
1633        }
1634        if let Err(e) = backend.update_status(&task.id, new_status).await {
1635            tracing::warn!(
1636                task_id,
1637                ?new_status,
1638                err = %e,
1639                "failed to mirror runner status to backend — store is authoritative"
1640            );
1641        }
1642
1643        let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ");
1644        let mut comment = if !summary.is_empty() {
1645            format!("[{now}] {status}: {summary}")
1646        } else if !last_error.is_empty() {
1647            format!("[{now}] {status}: {last_error}")
1648        } else {
1649            format!("[{now}] {status}")
1650        };
1651
1652        // Append visible attribution footer (matches PR body style)
1653        let footer = engine::attribution_footer("Created", &agent_name, model);
1654        comment.push_str(&footer);
1655
1656        backend.post_comment(&task.id, &comment).await?;
1657
1658        Ok(weight_signal)
1659    }
1660
1661    /// Process delegations from an agent response.
1662    ///
1663    /// Creates child GitHub issues for each delegation and marks the parent
1664    /// as blocked. The engine's Phase 4 unblock mechanism will re-activate
1665    /// the parent when all children are done.
1666    async fn process_delegations(
1667        &self,
1668        parent_task: &ExternalTask,
1669        delegations: &[Delegation],
1670        backend: &Arc<dyn ExternalBackend>,
1671    ) -> anyhow::Result<()> {
1672        let parent_id = &parent_task.id;
1673
1674        // Fetch existing open tasks once for dedup — prevents creating duplicate
1675        // GitHub issues when the same delegations are produced on repeated runs.
1676        let existing_titles: HashSet<String> = match backend.list_all_tasks().await {
1677            Ok(tasks) => tasks.into_iter().map(|t| t.title).collect(),
1678            Err(e) => {
1679                tracing::warn!(err = %e, "failed to fetch tasks for dedup — will create all");
1680                HashSet::new()
1681            }
1682        };
1683
1684        for delegation in delegations {
1685            // Skip if an issue with the same title already exists
1686            if existing_titles.contains(&delegation.title) {
1687                tracing::info!(
1688                    parent = parent_id.0,
1689                    title = delegation.title,
1690                    "skipping delegation — issue with same title already exists"
1691                );
1692                continue;
1693            }
1694
1695            // Build labels: status:new + any labels from the delegation
1696            let mut labels = delegation.labels.clone();
1697            labels.push("status:new".to_string());
1698
1699            // Build child body with delegation reference
1700            let child_body = format!(
1701                "{}\n\n---\n_Delegated from #{}_{}",
1702                delegation.body,
1703                parent_id.0,
1704                engine::orch_footer()
1705            );
1706
1707            match backend
1708                .create_sub_task(parent_id, &delegation.title, &child_body, &labels)
1709                .await
1710            {
1711                Ok(child_id) => {
1712                    tracing::info!(
1713                        parent = parent_id.0,
1714                        child = child_id.0,
1715                        title = delegation.title,
1716                        "created delegated subtask"
1717                    );
1718                }
1719                Err(e) => {
1720                    tracing::error!(
1721                        parent = parent_id.0,
1722                        title = delegation.title,
1723                        err = %e,
1724                        "failed to create delegated subtask"
1725                    );
1726                }
1727            }
1728        }
1729
1730        // Mark parent as blocked — store-first, then mirror to backend.
1731        if let Some(ref store) = self.store
1732            && let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &parent_id.0).await
1733        {
1734            store
1735                .update_status(store_id, TaskStatus::Blocked)
1736                .await
1737                .context("store-first: failed to block parent in store")?;
1738        }
1739        if let Err(e) = backend.update_status(parent_id, Status::Blocked).await {
1740            tracing::warn!(
1741                parent = parent_id.0,
1742                err = %e,
1743                "failed to mirror blocked status to backend — store is authoritative"
1744            );
1745        }
1746
1747        // Post summary comment on parent
1748        let summary = delegations
1749            .iter()
1750            .enumerate()
1751            .map(|(i, d)| format!("{}. {}", i + 1, d.title))
1752            .collect::<Vec<_>>()
1753            .join("\n");
1754
1755        backend
1756            .post_comment(
1757                parent_id,
1758                &format!(
1759                    "Delegated {} subtask(s):\n\n{}\n\nParent task is blocked until all subtasks complete.{}",
1760                    delegations.len(),
1761                    summary,
1762                    engine::orch_footer()
1763                ),
1764            )
1765            .await?;
1766
1767        Ok(())
1768    }
1769
1770    /// Resolve the project directory for this repo.
1771    pub(crate) async fn resolve_project_dir(&self) -> anyhow::Result<PathBuf> {
1772        // Explicit env var always wins
1773        if let Ok(dir) = env::var("PROJECT_DIR")
1774            && !dir.is_empty()
1775        {
1776            return Ok(PathBuf::from(dir));
1777        }
1778
1779        // Check projects list in global config — find path for this repo
1780        if let Ok(paths) = config::get_project_paths() {
1781            for path_str in &paths {
1782                let path = PathBuf::from(path_str);
1783                // Check if this project's .orch.yml has matching repo
1784                if let Ok(repo) = config::get_repo_for_project(&path) {
1785                    // Use tokio::fs::metadata for async-safe existence check
1786                    let path_for_check = path.clone();
1787                    if repo == self.repo && fs::metadata(&path_for_check).await.is_ok() {
1788                        return Ok(path);
1789                    }
1790                }
1791            }
1792        }
1793
1794        // Legacy: check config project_dir
1795        if let Ok(dir) = config::get("project_dir")
1796            && !dir.is_empty()
1797        {
1798            let path = PathBuf::from(&dir);
1799            let path_for_check = path.clone();
1800            if fs::metadata(&path_for_check).await.is_ok() {
1801                return Ok(path);
1802            }
1803        }
1804
1805        // Check for bare clone
1806        let parts: Vec<&str> = self.repo.split('/').collect();
1807        if parts.len() == 2 {
1808            let bare = self
1809                .orch_home
1810                .join("projects")
1811                .join(parts[0])
1812                .join(format!("{}.git", parts[1]));
1813            let bare_for_check = bare.clone();
1814            if task::spawn_blocking(move || bare_for_check.exists())
1815                .await
1816                .unwrap_or(false)
1817            {
1818                return Ok(bare);
1819            }
1820        }
1821
1822        // No resolvable project directory: not in active config, no legacy
1823        // project_dir, no bare clone. Falling back to the service's own CWD
1824        // here would silently point worktree creation at an unrelated
1825        // directory (see issue #3416) — surface a clear error instead.
1826        Err(NoProjectDir {
1827            repo: self.repo.clone(),
1828            bare_clone: self
1829                .orch_home
1830                .join("projects")
1831                .join(self.repo.split('/').next().unwrap_or_default())
1832                .join(format!(
1833                    "{}.git",
1834                    self.repo.split('/').nth(1).unwrap_or_default()
1835                )),
1836        }
1837        .into())
1838    }
1839}
1840
1841/// Update the GitHub agent label on an external task when it is rerouted to a new agent.
1842///
1843/// Removes the old `agent:<old_agent>` label, ensures the new `agent:<new_agent>` label exists
1844/// in the repo, then applies it. Returns `true` only when both `ensure_label` (repo label creation)
1845/// and `set_labels` (issue label assignment) succeed. `remove_label` failures are non-fatal.
1846async fn update_agent_label_after_reroute(
1847    #[cfg_attr(
1848        test,
1849        expect(unused_variables, reason = "only the label call outside tests uses it")
1850    )]
1851    repo: &str,
1852    task: &ExternalTask,
1853    backend: &Arc<dyn ExternalBackend>,
1854    old_agent: &str,
1855    new_agent: &str,
1856) -> bool {
1857    let old_label = format!("agent:{old_agent}");
1858    if let Err(e) = backend.remove_label(&task.id, &old_label).await {
1859        tracing::warn!(
1860            task_id = ?task.id,
1861            label = old_label,
1862            error = %e,
1863            "failed to remove old agent label during re-route"
1864        );
1865    }
1866    let new_label = format!("agent:{new_agent}");
1867
1868    // ensure_label creates the repo-level label if missing (best-effort).
1869    // Skipped under #[cfg(test)] to avoid real GitHub API calls.
1870    #[cfg(not(test))]
1871    {
1872        match GhHttp::new() {
1873            Ok(gh) => {
1874                if gh
1875                    .ensure_label(
1876                        repo,
1877                        &new_label,
1878                        http::status_label_color(&new_label),
1879                        &format!("Agent: {new_agent}"),
1880                    )
1881                    .await
1882                    .is_err()
1883                {
1884                    tracing::warn!(
1885                        task_id = ?task.id,
1886                        label = %new_label,
1887                        "ensure_label failed during agent label update after failover"
1888                    );
1889                }
1890            }
1891            Err(e) => {
1892                tracing::warn!(
1893                    task_id = ?task.id,
1894                    err = %e,
1895                    "GhHttp::new() failed — skipping ensure_label"
1896                );
1897            }
1898        }
1899    }
1900
1901    if let Err(e) = backend
1902        .set_labels(&task.id, slice::from_ref(&new_label))
1903        .await
1904    {
1905        tracing::warn!(
1906            task_id = ?task.id,
1907            label = %new_label,
1908            error = %e,
1909            "set_labels failed during agent label update after failover"
1910        );
1911        return false;
1912    }
1913    true
1914}
1915
1916/// Weight signal for a run that ended in `needs_review`. `prepare_task` clears `last_error` at run
1917/// start, so a non-empty value means `handle_error` ended this run
1918fn needs_review_signal(last_error: &str, agent: &str) -> WeightSignal {
1919    let agent = agent.to_string();
1920    if classify_run_error_type(last_error) == "rate_limit" {
1921        WeightSignal::RateLimited { agent }
1922    } else if !last_error.is_empty() {
1923        WeightSignal::Failed { agent }
1924    } else {
1925        WeightSignal::Success { agent }
1926    }
1927}
1928
1929#[cfg(test)]
1930mod tests {
1931    use super::*;
1932    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
1933    use crate::engine::runner::agents::patterns::safe_tail as safe_utf8_tail;
1934    use crate::parser::AgentResponse;
1935    use crate::store::{NewTask, TaskStatus, TaskStore};
1936    use async_trait::async_trait;
1937    use chrono::Utc;
1938    use std::sync::{Arc, Mutex};
1939    use tempfile::TempDir;
1940    use tokio::process::Command;
1941
1942    static ENV_LOCK: Mutex<()> = Mutex::new(());
1943
1944    fn ok_parse_result(status: &str) -> agents::ParsedResponse {
1945        agents::ParsedResponse {
1946            response: AgentResponse {
1947                status: status.to_string(),
1948                summary: String::new(),
1949                accomplished: vec![],
1950                remaining: vec![],
1951                files: vec![],
1952                error: None,
1953                input_tokens: None,
1954                output_tokens: None,
1955                learnings: vec![],
1956                delegations: vec![],
1957            },
1958            input_tokens: None,
1959            output_tokens: None,
1960            cost_usd: None,
1961            duration_ms: None,
1962        }
1963    }
1964
1965    // ── safe_utf8_tail ───────────────────────────────────────────────────────
1966
1967    #[test]
1968    fn safe_utf8_tail_short_string_returned_as_is() {
1969        assert_eq!(safe_utf8_tail("hello", 100), "hello");
1970        assert_eq!(safe_utf8_tail("", 10), "");
1971    }
1972
1973    #[test]
1974    fn safe_utf8_tail_truncates_ascii() {
1975        let s = "abcdefghij"; // 10 bytes
1976        assert_eq!(safe_utf8_tail(s, 5), "fghij");
1977        assert_eq!(safe_utf8_tail(s, 10), "abcdefghij");
1978    }
1979
1980    #[test]
1981    fn safe_utf8_tail_handles_multibyte_boundary() {
1982        // "日本語" is 3 chars × 3 bytes each = 9 bytes total
1983        let s = "日本語";
1984        assert_eq!(s.len(), 9);
1985        // Cutting at 8 bytes lands in the middle of "語" (3 bytes at 6..9).
1986        // safe_utf8_tail should walk forward to the next boundary at 9 (end) rather
1987        // than panicking on an invalid slice.
1988        let tail = safe_utf8_tail(s, 8);
1989        assert!(
1990            s.ends_with(tail),
1991            "tail must be a valid suffix of the original"
1992        );
1993        assert!(
1994            str::from_utf8(tail.as_bytes()).is_ok(),
1995            "tail must be valid UTF-8"
1996        );
1997    }
1998
1999    #[test]
2000    fn safe_utf8_tail_handles_mixed_ascii_and_multibyte() {
2001        let s = "hello日本語world"; // ascii + CJK + ascii
2002        for max_bytes in [1, 3, 5, 7, 11, 13, 20, 100] {
2003            let tail = safe_utf8_tail(s, max_bytes);
2004            assert!(
2005                str::from_utf8(tail.as_bytes()).is_ok(),
2006                "max_bytes={max_bytes}: tail must be valid UTF-8"
2007            );
2008            assert!(
2009                s.ends_with(tail),
2010                "max_bytes={max_bytes}: tail must be a suffix of original"
2011            );
2012        }
2013    }
2014
2015    #[test]
2016    fn safe_utf8_tail_exact_boundary() {
2017        let s = "abcdef"; // 6 bytes
2018        assert_eq!(safe_utf8_tail(s, 6), "abcdef"); // exactly fits
2019        assert_eq!(safe_utf8_tail(s, 0), ""); // empty tail
2020    }
2021
2022    #[test]
2023    fn parse_success_output_synthesizes_done_from_plain_text() {
2024        let runner = agents::get_runner("claude");
2025        let parsed = parse_success_output(
2026            "123",
2027            "claude",
2028            &*runner,
2029            "No open positions, no trade executions, no conditions change.",
2030        )
2031        .unwrap();
2032
2033        assert_eq!(parsed.response.status, "done");
2034        assert_eq!(
2035            parsed.response.summary,
2036            "No open positions, no trade executions, no conditions change."
2037        );
2038    }
2039
2040    #[test]
2041    fn parse_success_output_synthesizes_needs_review_from_error_text() {
2042        let runner = agents::get_runner("claude");
2043        let parsed =
2044            parse_success_output("123", "claude", &*runner, "Failed to update branch").unwrap();
2045
2046        assert_eq!(parsed.response.status, "needs_review");
2047        assert_eq!(
2048            parsed.response.error.as_deref(),
2049            Some("agent returned plain text instead of JSON")
2050        );
2051    }
2052
2053    #[test]
2054    fn parse_success_output_keeps_empty_text_invalid() {
2055        let runner = agents::get_runner("claude");
2056        let err = parse_success_output("123", "claude", &*runner, "").unwrap_err();
2057
2058        assert!(matches!(err, agents::AgentError::InvalidResponse { .. }));
2059    }
2060
2061    #[test]
2062    fn parse_success_output_preserves_tokens_from_ndjson() {
2063        // Simulates NDJSON output from --output-format stream-json where
2064        // the result event contains token usage. parse_response fails on
2065        // NDJSON (expects single JSON blob), but find_agent_result extracts
2066        // tokens from the NDJSON envelope so cost tracking is preserved.
2067        let ndjson = r#"{"type":"system","subtype":"init","session_id":"abc"}
2068{"type":"assistant","message":{"content":[{"type":"text","text":"Done."}],"usage":{"input_tokens":100,"output_tokens":50}}}
2069{"type":"result","subtype":"success","is_error":false,"duration_ms":5000,"result":"All changes committed successfully","usage":{"input_tokens":1234,"output_tokens":567}}"#;
2070        let runner = agents::get_runner("claude");
2071        let parsed = parse_success_output("456", "claude", &*runner, ndjson).unwrap();
2072
2073        assert_eq!(parsed.response.status, "done");
2074        assert_eq!(parsed.input_tokens, Some(1234));
2075        assert_eq!(parsed.output_tokens, Some(567));
2076        assert_eq!(parsed.duration_ms, Some(5000));
2077    }
2078
2079    #[test]
2080    fn parse_success_output_preserves_tokens_from_minimax_ndjson() {
2081        // MiniMax/Kimi use assistant event fallback when there's no result event.
2082        let ndjson = r#"{"type":"system","subtype":"init"}
2083{"type":"assistant","message":{"content":[{"type":"text","text":"No open positions."}],"usage":{"input_tokens":2000,"output_tokens":300}}}"#;
2084        let runner = agents::get_runner("minimax");
2085        let parsed = parse_success_output("789", "minimax", &*runner, ndjson).unwrap();
2086
2087        assert_eq!(parsed.response.status, "done");
2088        assert_eq!(parsed.input_tokens, Some(2000));
2089        assert_eq!(parsed.output_tokens, Some(300));
2090    }
2091
2092    #[test]
2093    fn classify_run_error_type_recognizes_specific_patterns() {
2094        let cases = [
2095            ("push failed: remote rejected", "push_failed"),
2096            ("No commits between main and branch", "no_commits"),
2097            ("create PR failed: 422", "pr_failed"),
2098            ("invalid response from agent", "parse_error"),
2099            ("parse error: missing json", "parse_error"),
2100            ("exceeded max attempts while rerouting", "max_attempts"),
2101            ("unit test failed", "ci_failure"),
2102            ("build failed: type mismatch", "ci_failure"),
2103            ("lint failed on warning", "ci_failure"),
2104            ("3 tests failed", "ci_failure"),
2105            ("test suite failed: 3 tests failed", "ci_failure"),
2106            (
2107                "claude interrupted: exited without a result envelope",
2108                "interrupted",
2109            ),
2110        ];
2111
2112        for (input, expected) in cases {
2113            assert_eq!(classify_run_error_type(input), expected, "input={input}");
2114        }
2115    }
2116
2117    #[test]
2118    fn classify_run_error_type_falls_back_for_unknown_errors() {
2119        assert_eq!(classify_run_error_type("something unexpected"), "failed");
2120    }
2121
2122    #[test]
2123    fn classify_run_error_type_connection_error_mentioning_pull_request_is_not_pr_failed() {
2124        // A generic GitHub API or connection error whose message happens to contain the
2125        // substring "pull request" must NOT be classified as pr_failed.
2126        assert_eq!(
2127            classify_run_error_type("failed to fetch pull request details from GitHub"),
2128            "failed"
2129        );
2130        assert_eq!(
2131            classify_run_error_type("connection reset while reading pull request data"),
2132            "failed"
2133        );
2134    }
2135
2136    #[test]
2137    fn classify_run_error_type_url_with_test_substring_is_not_ci_failure() {
2138        // A connection error whose URL path contains "/test-org/" must NOT be ci_failure.
2139        assert_eq!(
2140            classify_run_error_type(
2141                "failed to connect to api.github.com/repos/test-org/repo/pulls/123"
2142            ),
2143            "failed"
2144        );
2145        // Bare "test" word in an unrelated error must not trigger ci_failure.
2146        assert_eq!(
2147            classify_run_error_type("context deadline exceeded testing connection"),
2148            "failed"
2149        );
2150    }
2151
2152    #[test]
2153    fn classify_run_error_type_usage_without_limit_is_not_rate_limit() {
2154        // "usage" alone (e.g. token usage reporting) must NOT be rate_limit.
2155        assert_eq!(
2156            classify_run_error_type("token usage: 1245 tokens"),
2157            "failed"
2158        );
2159        assert_eq!(
2160            classify_run_error_type("incorrect usage of git command"),
2161            "failed"
2162        );
2163        // Only "usage limit" should trigger rate_limit.
2164        assert_eq!(
2165            classify_run_error_type("exceeded usage limit for this billing period"),
2166            "rate_limit"
2167        );
2168    }
2169
2170    #[test]
2171    fn classify_run_error_type_billing_cycle_is_not_auth_error() {
2172        // "billing cycle" quota-exhaustion messages must be classified as billing_cycle_exhausted,
2173        // not auth_error — aligning with cooldown.rs CreditExhaustionReason::BillingCycleExhausted.
2174        assert_eq!(
2175            classify_run_error_type(
2176                "You've reached your usage limit for this billing cycle. \
2177                 Your quota will be refreshed in the next cycle."
2178            ),
2179            "billing_cycle_exhausted"
2180        );
2181        assert_eq!(
2182            classify_run_error_type("quota exhausted for billing cycle"),
2183            "billing_cycle_exhausted"
2184        );
2185        // Plain "billing" without "cycle" that used to map to auth_error now falls through to "failed".
2186        assert_eq!(
2187            classify_run_error_type("billing account suspended"),
2188            "failed"
2189        );
2190    }
2191
2192    #[test]
2193    fn classify_run_error_type_monthly_quota_is_billing_cycle() {
2194        // Regression test for issue #3302: GitHub Copilot "You have exceeded your
2195        // monthly quota" must map to billing_cycle_exhausted, not "failed".
2196        assert_eq!(
2197            classify_run_error_type(
2198                "opencode failed: agent failed: You have exceeded your monthly quota"
2199            ),
2200            "billing_cycle_exhausted"
2201        );
2202        assert_eq!(
2203            classify_run_error_type("You have exceeded your monthly quota"),
2204            "billing_cycle_exhausted"
2205        );
2206    }
2207
2208    #[test]
2209    fn classify_run_outcome_routed_is_success() {
2210        // "routed" is a canonical non-success status (task will be re-dispatched).
2211        // It should be classified as "success" so the audit trail distinguishes it
2212        // from genuine failures (#2801).
2213        let parse_result = Ok(ok_parse_result("routed"));
2214        assert_eq!(
2215            classify_run_outcome("routed", &parse_result, false),
2216            "success"
2217        );
2218    }
2219
2220    #[test]
2221    fn classify_run_outcome_blocked_is_blocked() {
2222        let parse_result = Ok(ok_parse_result("blocked"));
2223        assert_eq!(
2224            classify_run_outcome("blocked", &parse_result, false),
2225            "blocked"
2226        );
2227    }
2228
2229    #[test]
2230    fn classify_run_outcome_non_canonical_is_failed() {
2231        // Non-canonical statuses should be classified as "failed" in the audit.
2232        // The explicit error message is provided by build_run_audit.
2233        let parse_result = Ok(ok_parse_result("fix_deployed"));
2234        assert_eq!(
2235            classify_run_outcome("fix_deployed", &parse_result, false),
2236            "failed"
2237        );
2238    }
2239
2240    #[test]
2241    fn classify_run_outcome_kimi_billing_cycle_is_not_rate_limit() {
2242        // Regression test for issue #3529: Kimi's exact billing-cycle exhaustion
2243        // message was stored as outcome="rate_limit" because the generic "usage
2244        // limit" pattern in detect_rate_limit() matched before credit-exhaustion
2245        // detection ever ran. The outcome must reflect the actual failure mode.
2246        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2247            Err(agents::AgentError::RateLimit {
2248                message: "Failed to authenticate. API Error: 403 You've reached your usage \
2249                          limit for this billing cycle. Your quota will be refreshed soon."
2250                    .to_string(),
2251            });
2252        assert_eq!(
2253            classify_run_outcome("needs_review", &parse_result, false),
2254            "billing_cycle_exhausted"
2255        );
2256    }
2257
2258    #[test]
2259    fn classify_run_outcome_generic_rate_limit_is_unaffected() {
2260        // A genuine rate limit (no billing-cycle/credit-exhaustion markers) must
2261        // still classify as "rate_limit".
2262        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2263            Err(agents::AgentError::RateLimit {
2264                message: "429 Too Many Requests".to_string(),
2265            });
2266        assert_eq!(
2267            classify_run_outcome("needs_review", &parse_result, false),
2268            "rate_limit"
2269        );
2270    }
2271
2272    #[test]
2273    fn classify_run_outcome_auth_billing_cycle_is_not_failed() {
2274        // Auth-classified billing-cycle exhaustion (e.g. "billing cycle exhausted"
2275        // matches detect_auth_error's pattern list) must also be distinguished from
2276        // a generic auth failure.
2277        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2278            Err(agents::AgentError::Auth {
2279                message: "billing cycle exhausted for this account".to_string(),
2280            });
2281        assert_eq!(
2282            classify_run_outcome("needs_review", &parse_result, false),
2283            "billing_cycle_exhausted"
2284        );
2285    }
2286
2287    #[test]
2288    fn classify_run_outcome_generic_auth_error_is_unaffected() {
2289        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2290            Err(agents::AgentError::Auth {
2291                message: "401 Unauthorized".to_string(),
2292            });
2293        assert_eq!(
2294            classify_run_outcome("needs_review", &parse_result, false),
2295            "failed"
2296        );
2297    }
2298
2299    #[test]
2300    fn classify_run_outcome_interrupted() {
2301        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2302            Err(agents::AgentError::Interrupted {
2303                message: "exited without a result envelope".to_string(),
2304            });
2305        assert_eq!(
2306            classify_run_outcome("new", &parse_result, false),
2307            "interrupted"
2308        );
2309    }
2310
2311    #[tokio::test]
2312    async fn build_run_audit_success_ignores_stale_last_error() {
2313        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2314        let task_id = store
2315            .create(&NewTask {
2316                repo: "owner/repo".to_string(),
2317                origin: "internal".to_string(),
2318                title: "Test".to_string(),
2319                ..Default::default()
2320            })
2321            .await
2322            .unwrap();
2323        store
2324            .set_fields(
2325                task_id,
2326                &[("last_error", serde_json::json!("no error info available"))],
2327            )
2328            .await
2329            .unwrap();
2330
2331        let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2332        let parse_result = Ok(ok_parse_result("done"));
2333        let started_at = Utc::now();
2334        let audit = runner
2335            .build_run_audit(RunAuditInput {
2336                task_id: &task_id.to_string(),
2337                status: "done",
2338                parse_result: &parse_result,
2339                raw_stdout: "",
2340                raw_stderr: "",
2341                started_at: &started_at,
2342                error_override: None,
2343                elapsed_secs: Some(1),
2344                push_failed: false,
2345            })
2346            .await;
2347
2348        assert_eq!(audit.outcome, "success");
2349        assert_eq!(audit.error, "");
2350    }
2351
2352    #[tokio::test]
2353    async fn reroute_clear_wipes_agent_when_it_still_matches_the_run() {
2354        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2355        let store_id = store
2356            .create(&NewTask {
2357                external_id: Some("42".to_string()),
2358                repo: "owner/repo".to_string(),
2359                origin: "github".to_string(),
2360                title: "Test".to_string(),
2361                ..Default::default()
2362            })
2363            .await
2364            .unwrap();
2365        store
2366            .update_status(store_id, TaskStatus::InProgress)
2367            .await
2368            .unwrap();
2369        store
2370            .set_fields(
2371                store_id,
2372                &[
2373                    ("agent", serde_json::json!("claude")),
2374                    ("model", serde_json::json!("sonnet")),
2375                ],
2376            )
2377            .await
2378            .unwrap();
2379
2380        let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store));
2381        runner
2382            .clear_agent_on_reroute_unless_fallback("42", "claude")
2383            .await;
2384
2385        let task = store.get(store_id).await.unwrap();
2386        assert!(
2387            task.agent.as_deref().unwrap_or("").is_empty(),
2388            "matching in-progress agent must be cleared for a normal reroute, got {:?}",
2389            task.agent
2390        );
2391        assert!(
2392            task.model.as_deref().unwrap_or("").is_empty(),
2393            "model must be cleared with the agent, got {:?}",
2394            task.model
2395        );
2396    }
2397
2398    #[tokio::test]
2399    async fn reroute_clear_keeps_a_different_in_progress_fallback() {
2400        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2401        let store_id = store
2402            .create(&NewTask {
2403                external_id: Some("42".to_string()),
2404                repo: "owner/repo".to_string(),
2405                origin: "github".to_string(),
2406                title: "Test".to_string(),
2407                ..Default::default()
2408            })
2409            .await
2410            .unwrap();
2411        store
2412            .update_status(store_id, TaskStatus::InProgress)
2413            .await
2414            .unwrap();
2415        store
2416            .set_fields(
2417                store_id,
2418                &[
2419                    ("agent", serde_json::json!("codex")),
2420                    ("model", serde_json::json!("")),
2421                ],
2422            )
2423            .await
2424            .unwrap();
2425
2426        let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store));
2427        runner
2428            .clear_agent_on_reroute_unless_fallback("42", "claude")
2429            .await;
2430
2431        let task = store.get(store_id).await.unwrap();
2432        assert_eq!(task.agent.as_deref(), Some("codex"));
2433        assert!(
2434            task.model.as_deref().unwrap_or("").is_empty(),
2435            "model stays empty so routing picks one for the fallback, got {:?}",
2436            task.model
2437        );
2438    }
2439
2440    #[tokio::test]
2441    async fn build_run_audit_ok_parse_prefers_parsed_error_over_stale_last_error() {
2442        // needs_review classifies as success and wipes the error. The #3851
2443        // misattribution is an Ok parse whose outcome is failed or blocked
2444        let cases = [("needs_action", "failed"), ("blocked", "blocked")];
2445        let parsed_error = "passbox Touch ID prompt timed out for cf/read-token";
2446        let stale = "minimax auth error: API Error: 402 insufficient balance (1008), clearing agent/model for re-route";
2447        for (status, expected_outcome) in cases {
2448            let store = Arc::new(TaskStore::open_memory().await.unwrap());
2449            let task_id = store
2450                .create(&NewTask {
2451                    repo: "owner/repo".to_string(),
2452                    origin: "internal".to_string(),
2453                    title: "Test".to_string(),
2454                    ..Default::default()
2455                })
2456                .await
2457                .unwrap();
2458            store
2459                .set_fields(task_id, &[("last_error", serde_json::json!(stale))])
2460                .await
2461                .unwrap();
2462
2463            let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2464            let parse_result = Ok(agents::ParsedResponse {
2465                response: AgentResponse {
2466                    status: status.to_string(),
2467                    summary: "cache_stats.py did not run".to_string(),
2468                    accomplished: vec![],
2469                    remaining: vec![],
2470                    files: vec![],
2471                    error: Some(parsed_error.to_string()),
2472                    input_tokens: None,
2473                    output_tokens: None,
2474                    learnings: vec![],
2475                    delegations: vec![],
2476                },
2477                input_tokens: None,
2478                output_tokens: None,
2479                cost_usd: None,
2480                duration_ms: None,
2481            });
2482            let started_at = Utc::now();
2483            let audit = runner
2484                .build_run_audit(RunAuditInput {
2485                    task_id: &task_id.to_string(),
2486                    status,
2487                    parse_result: &parse_result,
2488                    raw_stdout: "",
2489                    raw_stderr: "",
2490                    started_at: &started_at,
2491                    error_override: None,
2492                    elapsed_secs: Some(261),
2493                    push_failed: false,
2494                })
2495                .await;
2496
2497            assert_eq!(audit.outcome, expected_outcome, "status={status}");
2498            assert_eq!(audit.error, parsed_error, "status={status}");
2499        }
2500    }
2501
2502    #[tokio::test]
2503    async fn build_run_audit_non_success_uses_fallback_error_when_empty() {
2504        let runner = TaskRunner::new("owner/repo".to_string());
2505        let parse_result = Ok(ok_parse_result("new"));
2506        let started_at = Utc::now();
2507        let audit = runner
2508            .build_run_audit(RunAuditInput {
2509                task_id: "1",
2510                status: "new",
2511                parse_result: &parse_result,
2512                raw_stdout: "",
2513                raw_stderr: "",
2514                started_at: &started_at,
2515                error_override: None,
2516                elapsed_secs: Some(1),
2517                push_failed: false,
2518            })
2519            .await;
2520
2521        assert_eq!(audit.outcome, "failed");
2522        // "new" is a canonical non-success status — no error message needed.
2523        assert_eq!(audit.error, "");
2524    }
2525
2526    #[tokio::test]
2527    async fn build_run_audit_non_canonical_status_reports_explicit_error() {
2528        // Non-canonical statuses (not in the allowlist) with no parse error
2529        // should get an explicit "unrecognized status: X" message instead of
2530        // the opaque "no error info available" placeholder (#2801).
2531        let runner = TaskRunner::new("owner/repo".to_string());
2532        let parse_result = Ok(ok_parse_result("fix_deployed"));
2533        let started_at = Utc::now();
2534        let audit = runner
2535            .build_run_audit(RunAuditInput {
2536                task_id: "1",
2537                status: "fix_deployed",
2538                parse_result: &parse_result,
2539                raw_stdout: "",
2540                raw_stderr: "",
2541                started_at: &started_at,
2542                error_override: None,
2543                elapsed_secs: Some(1),
2544                push_failed: false,
2545            })
2546            .await;
2547
2548        assert_eq!(audit.outcome, "failed");
2549        assert_eq!(audit.error, "unrecognized status: fix_deployed");
2550    }
2551
2552    #[tokio::test]
2553    async fn build_run_audit_canonical_non_success_status_no_error_message() {
2554        // Canonical non-success statuses (e.g. "routed") should not produce
2555        // an error message — they're expected outcomes, not failures.
2556        for status in &["routed"] {
2557            let runner = TaskRunner::new("owner/repo".to_string());
2558            let parse_result = Ok(ok_parse_result(status));
2559            let started_at = Utc::now();
2560            let audit = runner
2561                .build_run_audit(RunAuditInput {
2562                    task_id: "1",
2563                    status,
2564                    parse_result: &parse_result,
2565                    raw_stdout: "",
2566                    raw_stderr: "",
2567                    started_at: &started_at,
2568                    error_override: None,
2569                    elapsed_secs: Some(1),
2570                    push_failed: false,
2571                })
2572                .await;
2573
2574            assert_eq!(audit.outcome, "success", "status={status}");
2575            assert_eq!(
2576                audit.error, "",
2577                "status={status}: no error for canonical status"
2578            );
2579        }
2580    }
2581
2582    #[tokio::test]
2583    async fn build_run_audit_blocked_uses_summary_when_error_missing() {
2584        let runner = TaskRunner::new("owner/repo".to_string());
2585        let parse_result = Ok(agents::ParsedResponse {
2586            response: AgentResponse {
2587                status: "blocked".to_string(),
2588                summary: "Could not proceed due to missing credentials".to_string(),
2589                accomplished: vec![],
2590                remaining: vec![],
2591                files: vec![],
2592                error: None,
2593                input_tokens: None,
2594                output_tokens: None,
2595                learnings: vec![],
2596                delegations: vec![],
2597            },
2598            input_tokens: None,
2599            output_tokens: None,
2600            cost_usd: None,
2601            duration_ms: None,
2602        });
2603        let started_at = Utc::now();
2604        let audit = runner
2605            .build_run_audit(RunAuditInput {
2606                task_id: "1",
2607                status: "blocked",
2608                parse_result: &parse_result,
2609                raw_stdout: "",
2610                raw_stderr: "",
2611                started_at: &started_at,
2612                error_override: None,
2613                elapsed_secs: Some(1),
2614                push_failed: false,
2615            })
2616            .await;
2617
2618        assert_eq!(audit.outcome, "blocked");
2619        assert_eq!(audit.error, "Could not proceed due to missing credentials");
2620    }
2621
2622    // ── resolve_project_dir ──────────────────────────────────────────────────
2623
2624    #[tokio::test]
2625    async fn resolve_project_dir_uses_project_dir_env() {
2626        // When PROJECT_DIR env var is set, it should always win.
2627        let runner = TaskRunner::new("owner/repo".to_string());
2628
2629        // Use a temp dir so the path exists
2630        let dir = tempfile::tempdir().unwrap();
2631        let dir_str = dir.path().to_string_lossy().to_string();
2632
2633        // Temporarily set the env var — note: tests run in parallel so we use a
2634        // dedicated key that only this test touches. std::env::set_var is not
2635        // thread-safe in general, but the subsequent call is synchronous and
2636        // isolated to the temp dir created above.
2637        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2638        unsafe { env::set_var("PROJECT_DIR", &dir_str) };
2639        let result = runner.resolve_project_dir().await;
2640        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2641        unsafe { env::remove_var("PROJECT_DIR") };
2642
2643        assert!(
2644            result.is_ok(),
2645            "resolve_project_dir should succeed with PROJECT_DIR set"
2646        );
2647        assert_eq!(result.unwrap(), dir.path());
2648    }
2649
2650    #[tokio::test]
2651    async fn resolve_project_dir_empty_project_dir_env_falls_through() {
2652        // An empty PROJECT_DIR should be ignored and fall through to other logic.
2653        let runner = TaskRunner::new("owner/testrepo-nonexistent".to_string());
2654        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2655        unsafe { env::set_var("PROJECT_DIR", "") };
2656        let result = runner.resolve_project_dir().await;
2657        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2658        unsafe { env::remove_var("PROJECT_DIR") };
2659
2660        // No active config entry, no legacy project_dir, no bare clone for this
2661        // repo: should return an explicit error rather than falling back to the
2662        // service's own CWD (see issue #3416).
2663        assert!(
2664            result.is_err(),
2665            "resolve_project_dir should error when no project dir can be resolved"
2666        );
2667        let err = result.unwrap_err().to_string();
2668        assert!(
2669            err.contains("owner/testrepo-nonexistent"),
2670            "error should name the unresolvable repo: {err}"
2671        );
2672    }
2673
2674    // ── mock backend for process_delegations ─────────────────────────────────
2675
2676    struct TrackingBackend {
2677        sub_tasks_created: Arc<Mutex<Vec<(String, String)>>>, // (title, parent_id)
2678        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
2679        comments: Arc<Mutex<Vec<(String, String)>>>, // (id, body)
2680    }
2681
2682    impl TrackingBackend {
2683        fn new() -> Self {
2684            Self {
2685                sub_tasks_created: Arc::new(Mutex::new(vec![])),
2686                status_updates: Arc::new(Mutex::new(vec![])),
2687                comments: Arc::new(Mutex::new(vec![])),
2688            }
2689        }
2690    }
2691
2692    #[async_trait]
2693    impl ExternalBackend for TrackingBackend {
2694        fn name(&self) -> &'static str {
2695            "tracking"
2696        }
2697        async fn create_task(
2698            &self,
2699            _t: &str,
2700            _b: &str,
2701            _l: &[String],
2702        ) -> anyhow::Result<ExternalId> {
2703            Ok(ExternalId("new".to_string()))
2704        }
2705        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2706            Ok(ExternalTask {
2707                id: id.clone(),
2708                title: String::new(),
2709                body: String::new(),
2710                state: "open".to_string(),
2711                labels: vec![],
2712                author: String::new(),
2713                created_at: "2026-01-01T00:00:00Z".to_string(),
2714                updated_at: "2026-01-01T00:00:00Z".to_string(),
2715                url: String::new(),
2716            })
2717        }
2718        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
2719            Ok(vec![])
2720        }
2721        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2722            Ok(vec![])
2723        }
2724        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2725            self.comments
2726                .lock()
2727                .unwrap()
2728                .push((id.0.clone(), body.to_string()));
2729            Ok(())
2730        }
2731        async fn set_labels(&self, _id: &ExternalId, _l: &[String]) -> anyhow::Result<()> {
2732            Ok(())
2733        }
2734        async fn remove_label(&self, _id: &ExternalId, _l: &str) -> anyhow::Result<()> {
2735            Ok(())
2736        }
2737        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2738            Ok(vec![])
2739        }
2740        async fn create_sub_task(
2741            &self,
2742            parent: &ExternalId,
2743            title: &str,
2744            _body: &str,
2745            _l: &[String],
2746        ) -> anyhow::Result<ExternalId> {
2747            self.sub_tasks_created
2748                .lock()
2749                .unwrap()
2750                .push((title.to_string(), parent.0.clone()));
2751            Ok(ExternalId(format!("child-{title}")))
2752        }
2753        async fn ensure_status_label(&self, _l: &str) -> anyhow::Result<()> {
2754            Ok(())
2755        }
2756        async fn has_open_issue_with_title(&self, _t: &str, _l: &str) -> anyhow::Result<bool> {
2757            Ok(false)
2758        }
2759        async fn health_check(&self) -> anyhow::Result<()> {
2760            Ok(())
2761        }
2762        async fn is_pr_merged(&self, _b: &str) -> anyhow::Result<bool> {
2763            Ok(false)
2764        }
2765        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2766            Ok(None)
2767        }
2768        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
2769            Ok(vec![])
2770        }
2771        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2772            self.status_updates
2773                .lock()
2774                .unwrap()
2775                .push((id.0.clone(), status));
2776            Ok(())
2777        }
2778    }
2779
2780    fn make_task(id: &str) -> ExternalTask {
2781        ExternalTask {
2782            id: ExternalId(id.to_string()),
2783            title: format!("Task {id}"),
2784            body: String::new(),
2785            state: "open".to_string(),
2786            labels: vec![],
2787            author: "bot".to_string(),
2788            created_at: "2026-01-01T00:00:00Z".to_string(),
2789            updated_at: "2026-01-01T00:00:00Z".to_string(),
2790            url: String::new(),
2791        }
2792    }
2793
2794    // ── process_delegations ───────────────────────────────────────────────────
2795
2796    #[expect(
2797        clippy::await_holding_lock,
2798        reason = "the env lock must stay held across the awaited calls"
2799    )]
2800    #[tokio::test]
2801    async fn process_delegations_creates_subtasks_and_blocks_parent() {
2802        let _guard = ENV_LOCK.lock().unwrap();
2803        let temp_home = TempDir::new().unwrap();
2804        let orch_home = temp_home.path().join(".orch");
2805        fs::create_dir_all(&orch_home).await.unwrap();
2806        let old_orch_home = env::var("ORCH_HOME").ok();
2807        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2808        unsafe { env::set_var("ORCH_HOME", &orch_home) };
2809
2810        let runner = TaskRunner::new("owner/repo".to_string());
2811        let parent = make_task("99");
2812        let backend = Arc::new(TrackingBackend::new());
2813        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<TrackingBackend>::clone(&backend);
2814
2815        let delegations = vec![
2816            Delegation {
2817                title: "Subtask A".to_string(),
2818                body: "Do A".to_string(),
2819                labels: vec!["enhancement".to_string()],
2820            },
2821            Delegation {
2822                title: "Subtask B".to_string(),
2823                body: "Do B".to_string(),
2824                labels: vec![],
2825            },
2826        ];
2827
2828        runner
2829            .process_delegations(&parent, &delegations, &backend_dyn)
2830            .await
2831            .unwrap();
2832
2833        // Two sub-tasks should have been created
2834        let created = backend.sub_tasks_created.lock().unwrap();
2835        assert_eq!(created.len(), 2);
2836        assert_eq!(created[0].1, "99", "parent id should be 99");
2837        assert!(
2838            created.iter().any(|(t, _)| t == "Subtask A"),
2839            "Subtask A should be created"
2840        );
2841        assert!(
2842            created.iter().any(|(t, _)| t == "Subtask B"),
2843            "Subtask B should be created"
2844        );
2845        drop(created);
2846
2847        // Parent should be marked blocked
2848        let updates = backend.status_updates.lock().unwrap();
2849        assert!(
2850            updates
2851                .iter()
2852                .any(|(id, s)| id == "99" && *s == Status::Blocked),
2853            "parent should be blocked"
2854        );
2855        drop(updates);
2856
2857        // A comment summarising the delegations should be posted
2858        let comments = backend.comments.lock().unwrap();
2859        assert!(
2860            !comments.is_empty(),
2861            "a delegation summary comment should be posted"
2862        );
2863        let comment_body = &comments[0].1;
2864        assert!(
2865            comment_body.contains("Subtask A"),
2866            "summary should mention Subtask A"
2867        );
2868        assert!(
2869            comment_body.contains("Subtask B"),
2870            "summary should mention Subtask B"
2871        );
2872
2873        if let Some(old) = old_orch_home {
2874            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2875            unsafe { env::set_var("ORCH_HOME", old) };
2876        } else {
2877            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2878            unsafe { env::remove_var("ORCH_HOME") };
2879        }
2880    }
2881
2882    #[expect(
2883        clippy::await_holding_lock,
2884        reason = "the env lock must stay held across the awaited calls"
2885    )]
2886    #[tokio::test]
2887    async fn process_delegations_single_subtask() {
2888        let _guard = ENV_LOCK.lock().unwrap();
2889        let temp_home = TempDir::new().unwrap();
2890        let orch_home = temp_home.path().join(".orch");
2891        fs::create_dir_all(&orch_home).await.unwrap();
2892        let old_orch_home = env::var("ORCH_HOME").ok();
2893        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2894        unsafe { env::set_var("ORCH_HOME", &orch_home) };
2895
2896        let runner = TaskRunner::new("owner/repo".to_string());
2897        let parent = make_task("101");
2898        let backend = Arc::new(TrackingBackend::new());
2899        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<TrackingBackend>::clone(&backend);
2900
2901        let delegations = vec![Delegation {
2902            title: "Only Child".to_string(),
2903            body: "Do only child".to_string(),
2904            labels: vec!["feature".to_string()],
2905        }];
2906
2907        runner
2908            .process_delegations(&parent, &delegations, &backend_dyn)
2909            .await
2910            .unwrap();
2911
2912        let created = backend.sub_tasks_created.lock().unwrap();
2913        assert_eq!(created.len(), 1);
2914        assert_eq!(created[0].0, "Only Child");
2915        drop(created);
2916
2917        let updates = backend.status_updates.lock().unwrap();
2918        assert!(
2919            updates
2920                .iter()
2921                .any(|(id, s)| id == "101" && *s == Status::Blocked),
2922            "parent should be blocked"
2923        );
2924        drop(updates);
2925
2926        let comments = backend.comments.lock().unwrap();
2927        let body = &comments[0].1;
2928        assert!(
2929            body.contains("1 subtask"),
2930            "comment should count one subtask"
2931        );
2932
2933        if let Some(old) = old_orch_home {
2934            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2935            unsafe { env::set_var("ORCH_HOME", old) };
2936        } else {
2937            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2938            unsafe { env::remove_var("ORCH_HOME") };
2939        }
2940    }
2941
2942    // ── weight signal logic ───────────────────────────────────────────────────
2943
2944    /// Test helper that mirrors the weight signal logic from `run_with_context`.
2945    /// Reroutes only emit `RateLimited` when the error type is genuinely "rate_limit".
2946    fn weight_signal_for(status: &str, error_type: &str, agent: &str) -> WeightSignal {
2947        let is_rerouted = status == "new" || status == "routed";
2948        let is_rate_limit_error = error_type == "rate_limit";
2949        if is_rerouted {
2950            if is_rate_limit_error {
2951                WeightSignal::RateLimited {
2952                    agent: agent.to_string(),
2953                }
2954            } else {
2955                WeightSignal::Rerouted
2956            }
2957        } else if status == "done"
2958            || status == "needs_review"
2959            || status == "in_progress"
2960            || status == "in_review"
2961        {
2962            WeightSignal::Success {
2963                agent: agent.to_string(),
2964            }
2965        } else if status == "blocked" {
2966            WeightSignal::Blocked
2967        } else {
2968            WeightSignal::None
2969        }
2970    }
2971
2972    #[test]
2973    fn weight_signal_success_includes_needs_review() {
2974        // The normal happy-path for code tasks: agent reports done, has a PR,
2975        // so handle_success returns "needs_review". This must map to Success.
2976        let signal = weight_signal_for("needs_review", "success", "claude");
2977        assert!(
2978            matches!(signal, WeightSignal::Success { agent } if agent == "claude"),
2979            "needs_review should produce WeightSignal::Success"
2980        );
2981    }
2982
2983    #[test]
2984    fn needs_review_signal_success_for_empty_error() {
2985        let signal = needs_review_signal("", "claude");
2986        assert!(matches!(signal, WeightSignal::Success { agent } if agent == "claude"));
2987    }
2988
2989    #[test]
2990    fn needs_review_signal_failed_after_timeout_with_no_fallback() {
2991        let signal =
2992            needs_review_signal("claude timed out after 1804s, no fallback agents", "claude");
2993        assert!(matches!(signal, WeightSignal::Failed { agent } if agent == "claude"));
2994    }
2995
2996    #[test]
2997    fn needs_review_signal_rate_limited_for_rate_limit() {
2998        let signal = needs_review_signal("rate limit exceeded, try again later", "claude");
2999        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3000    }
3001
3002    #[test]
3003    fn weight_signal_success_for_all_done_statuses() {
3004        for status in ["done", "needs_review", "in_progress", "in_review"] {
3005            let signal = weight_signal_for(status, "success", "codex");
3006            assert!(
3007                matches!(signal, WeightSignal::Success { agent } if agent == "codex"),
3008                "{status} should produce WeightSignal::Success"
3009            );
3010        }
3011    }
3012
3013    #[test]
3014    fn weight_signal_rate_limited_for_reroute_with_rate_limit_error() {
3015        // Reroute due to rate limit should emit RateLimited.
3016        let signal = weight_signal_for("new", "rate_limit", "claude");
3017        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3018    }
3019
3020    #[test]
3021    fn weight_signal_rate_limited_for_routed_status_with_rate_limit_error() {
3022        let signal = weight_signal_for("routed", "rate_limit", "claude");
3023        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3024    }
3025
3026    #[test]
3027    fn weight_signal_rerouted_for_non_rate_limit_reroute() {
3028        // Reroute due to silence detection, timeout, or other non-rate-limit errors
3029        // should emit Rerouted (not RateLimited and not None).
3030        // Rerouted maps to "new" in tick so the task stays retryable and is never
3031        // converted to needs_review → done via the no-code review shortcut.
3032        for error_type in ["timeout", "silence detected", "parse_error", "failed"] {
3033            for status in ["new", "routed"] {
3034                let signal = weight_signal_for(status, error_type, "claude");
3035                assert!(
3036                    matches!(signal, WeightSignal::Rerouted),
3037                    "{status} with error_type={error_type} should produce WeightSignal::Rerouted, not RateLimited or None"
3038                );
3039            }
3040        }
3041    }
3042
3043    #[test]
3044    fn silence_reset_rerouted_never_becomes_needs_review() {
3045        // Regression test for: silence detection sets task to new, runner returns
3046        // WeightSignal::Rerouted, tick must map that to "new" — not "needs_review".
3047        // If it mapped to needs_review the no-code shortcut would close the task as
3048        // done with no successful agent run in the audit trail.
3049        let signal = weight_signal_for("new", "silence detected", "claude");
3050        assert!(
3051            matches!(signal, WeightSignal::Rerouted),
3052            "silence-reset reroute must produce Rerouted, never None/needs_review"
3053        );
3054        // Verify tick mapping: Rerouted => "new" (not needs_review or done)
3055        let display_status = match &signal {
3056            WeightSignal::Rerouted => "new",
3057            WeightSignal::None => "needs_review", // the old (buggy) path
3058            _ => "other",
3059        };
3060        assert_eq!(
3061            display_status, "new",
3062            "Rerouted must map to 'new' in tick, not 'needs_review'"
3063        );
3064    }
3065
3066    #[test]
3067    fn weight_signal_blocked_status() {
3068        let signal = weight_signal_for("blocked", "success", "claude");
3069        assert!(matches!(signal, WeightSignal::Blocked));
3070    }
3071
3072    // ── parse_success_output token extraction ────────────────────────────────
3073
3074    /// Mock runner that always returns InvalidResponse with truncated text,
3075    /// simulating what minimax/codex do when they can't parse structured JSON.
3076    struct FailingMockRunner;
3077
3078    impl agents::AgentRunner for FailingMockRunner {
3079        #[cfg(test)]
3080        fn name(&self) -> &'static str {
3081            "opencode"
3082        }
3083
3084        fn stream_events(&self, _event: &serde_json::Value) -> Vec<agents::StreamEvent> {
3085            vec![]
3086        }
3087
3088        fn build_command(
3089            &self,
3090            _model: Option<&str>,
3091            _timeout_cmd: &str,
3092            _sys_file: &str,
3093            _msg_file: &str,
3094            _permissions: &agents::PermissionRules,
3095        ) -> String {
3096            String::new()
3097        }
3098
3099        fn parse_response(&self, raw: &str) -> Result<agents::ParsedResponse, agents::AgentError> {
3100            // Simulate what minimax does: truncate to 300 chars
3101            Err(agents::AgentError::InvalidResponse {
3102                raw: raw.chars().take(30).collect(),
3103            })
3104        }
3105
3106        fn extract_text(&self, raw: &str) -> Result<String, agents::AgentError> {
3107            Err(agents::AgentError::InvalidResponse {
3108                raw: raw.to_string(),
3109            })
3110        }
3111
3112        fn classify_error(
3113            &self,
3114            exit_code: i32,
3115            _stdout: &str,
3116            _stderr: &str,
3117        ) -> agents::AgentError {
3118            agents::AgentError::Unknown {
3119                exit_code,
3120                message: "mock error".into(),
3121            }
3122        }
3123
3124        fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
3125            anyhow::bail!("not implemented for mock")
3126        }
3127    }
3128
3129    #[test]
3130    fn parse_success_output_extracts_tokens_from_raw_stdout_not_error_raw() {
3131        // OpenCode NDJSON output with token metadata in step_finish
3132        let ndjson = r#"{"type":"text","timestamp":1,"part":{"type":"text","text":"Here is my response: {\"status\":\"done\",\"summary\":\"fixed\"}}"}}
3133{"type":"step_finish","timestamp":2,"part":{"type":"step-finish","reason":"stop","cost":0,"tokens":{"total":95177,"input":94920,"output":257}}}"#;
3134
3135        let runner = FailingMockRunner;
3136        let result = parse_success_output("1550", "opencode", &runner, ndjson);
3137
3138        // Should succeed (synthesized from text) with tokens preserved
3139        let parsed = result.expect("should synthesize response from text");
3140        assert_eq!(parsed.input_tokens, Some(94920));
3141        assert_eq!(parsed.output_tokens, Some(257));
3142    }
3143
3144    #[test]
3145    fn parse_success_output_no_tokens_when_ndjson_has_none() {
3146        // Plain text with an explicit completion signal but no token metadata.
3147        // Uses "No open positions" which is in the explicit_done list.
3148        let text = "No open positions, no trade executions, no conditions change.";
3149
3150        let runner = FailingMockRunner;
3151        let result = parse_success_output("1551", "opencode", &runner, text);
3152
3153        // Should synthesize a "done" response, but without tokens (no NDJSON envelope).
3154        let parsed = result.expect("should synthesize response from explicit-done text");
3155        assert_eq!(parsed.input_tokens, None);
3156        assert_eq!(parsed.output_tokens, None);
3157    }
3158
3159    #[test]
3160    fn parse_success_output_uses_agent_result_directly() {
3161        // Here we test that if find_agent_result succeeds and returns valid JSON, we use it directly
3162        // WITHOUT calling agent_runner.parse_response at all.
3163        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"status\":\"done\",\"summary\":\"my summary\",\"accomplished\":[],\"remaining\":[],\"files\":[]}"}"#;
3164
3165        // FailingMockRunner always returns InvalidResponse, so if we called it, we'd synthesize "done" instead of getting "my summary".
3166        let runner = FailingMockRunner;
3167        let result = parse_success_output("999", "claude", &runner, ndjson);
3168
3169        let parsed = result.expect("should parse successfully via find_agent_result");
3170        assert_eq!(parsed.response.status, "done");
3171        assert_eq!(parsed.response.summary, "my summary");
3172    }
3173
3174    #[test]
3175    fn parse_success_output_treats_agent_result_is_error_as_authoritative() {
3176        // Here we test that if find_agent_result returns is_error=true, we return AgentFailed
3177        // instead of falling back to synthesis or agent_runner.parse_response.
3178        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"I failed because of reasons"}"#;
3179
3180        let runner = FailingMockRunner;
3181        let result = parse_success_output("999", "claude", &runner, ndjson);
3182
3183        let err = result.expect_err("should return error");
3184        match err {
3185            agents::AgentError::AgentFailed { message } => {
3186                assert_eq!(message, "I failed because of reasons");
3187            }
3188            _ => panic!("expected AgentFailed, got {err:?}"),
3189        }
3190    }
3191
3192    #[test]
3193    fn parse_success_output_classifies_model_unavailable_from_is_error_result() {
3194        // Regression test for issue #3607: the is_error=true NDJSON path
3195        // (used by codex's find_codex_result, which stringifies its own
3196        // classified AgentError into result_text) must re-classify a
3197        // model-not-found message as ModelUnavailable so it gets
3198        // model-scoped cooldown treatment instead of degrading to a
3199        // generic AgentFailed, which triggers an agent-wide cooldown.
3200        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"model unavailable (gpt-5.5): Reconnecting... 2/5 (unexpected status 404 Not Found: The model `gpt-5.5` does not exist or you do not have access to it.)"}"#;
3201
3202        let runner = FailingMockRunner;
3203        let result = parse_success_output("999", "claude", &runner, ndjson);
3204
3205        let err = result.expect_err("should return error");
3206        match err {
3207            agents::AgentError::ModelUnavailable { model, .. } => {
3208                assert_eq!(model, "gpt-5.5");
3209            }
3210            _ => panic!("expected ModelUnavailable, got {err:?}"),
3211        }
3212    }
3213
3214    #[test]
3215    fn nonzero_exit_with_ndjson_success_is_parsed_as_success() {
3216        // Simulate an agent that exits with non-zero code but emits NDJSON result
3217        // with is_error=false and a JSON result payload. The runner should accept
3218        // this as success because the NDJSON result is authoritative.
3219        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1234,"result":"{\"status\":\"done\",\"summary\":\"All good\"}","usage":{"input_tokens":10,"output_tokens":5}}"#;
3220
3221        // Use a runner that would normally fail parsing to ensure we're using
3222        // the find_agent_result path. FailingMockRunner.parse_response returns
3223        // InvalidResponse, but parse_success_output should short-circuit.
3224        let runner = FailingMockRunner;
3225
3226        // Construct a fake SessionOutput
3227        let session_output = session::SessionOutput {
3228            exit_code: 1,
3229            raw_stdout: ndjson.to_string(),
3230            raw_stderr: String::new(),
3231            elapsed_secs: Some(1),
3232            sandbox_error: None,
3233        };
3234
3235        // Call the helper we added via the task runner logic by reusing the
3236        // parse_success_output directly since it encapsulates the same behavior.
3237        let parsed =
3238            parse_success_output("task-ndjson", "claude", &runner, &session_output.raw_stdout)
3239                .expect("should parse NDJSON success");
3240        assert_eq!(parsed.response.status, "done");
3241        assert_eq!(parsed.response.summary, "All good");
3242        assert_eq!(parsed.input_tokens, Some(10));
3243        assert_eq!(parsed.output_tokens, Some(5));
3244    }
3245
3246    // ── agent label update after reroute ─────────────────────────────────────
3247
3248    /// Backend that wraps TrackingBackend but can be configured to fail label ops.
3249    struct LabelFailingBackend {
3250        inner: TrackingBackend,
3251        /// If set, set_labels returns this error.
3252        set_labels_err: Option<String>,
3253    }
3254
3255    impl LabelFailingBackend {
3256        fn new() -> Self {
3257            Self {
3258                inner: TrackingBackend::new(),
3259                set_labels_err: None,
3260            }
3261        }
3262        fn with_set_labels_err(mut self, err: impl Into<String>) -> Self {
3263            self.set_labels_err = Some(err.into());
3264            self
3265        }
3266    }
3267
3268    impl Default for LabelFailingBackend {
3269        fn default() -> Self {
3270            Self::new()
3271        }
3272    }
3273
3274    #[async_trait]
3275    impl ExternalBackend for LabelFailingBackend {
3276        fn name(&self) -> &'static str {
3277            "label_failing"
3278        }
3279        async fn create_task(
3280            &self,
3281            _t: &str,
3282            _b: &str,
3283            _l: &[String],
3284        ) -> anyhow::Result<ExternalId> {
3285            Ok(ExternalId("new".to_string()))
3286        }
3287        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3288            self.inner.get_task(id).await
3289        }
3290        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
3291            Ok(vec![])
3292        }
3293        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3294            Ok(vec![])
3295        }
3296        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3297            self.inner.post_comment(id, body).await
3298        }
3299        async fn set_labels(&self, id: &ExternalId, labels: &[String]) -> anyhow::Result<()> {
3300            if let Some(ref err) = self.set_labels_err {
3301                return Err(anyhow::anyhow!("{err}"));
3302            }
3303            self.inner.set_labels(id, labels).await
3304        }
3305        async fn remove_label(&self, id: &ExternalId, label: &str) -> anyhow::Result<()> {
3306            self.inner.remove_label(id, label).await
3307        }
3308        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3309            Ok(vec![])
3310        }
3311        async fn create_sub_task(
3312            &self,
3313            parent: &ExternalId,
3314            title: &str,
3315            body: &str,
3316            labels: &[String],
3317        ) -> anyhow::Result<ExternalId> {
3318            self.inner
3319                .create_sub_task(parent, title, body, labels)
3320                .await
3321        }
3322        async fn ensure_status_label(&self, label: &str) -> anyhow::Result<()> {
3323            self.inner.ensure_status_label(label).await
3324        }
3325        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3326            self.inner.update_status(id, status).await
3327        }
3328        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3329            self.inner.get_authenticated_user().await
3330        }
3331        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
3332            Ok(vec![])
3333        }
3334        async fn health_check(&self) -> anyhow::Result<()> {
3335            self.inner.health_check().await
3336        }
3337    }
3338
3339    /// `update_agent_label_after_reroute` returns `false` when `set_labels` fails.
3340    ///
3341    /// This tests the extracted helper directly, bypassing the full `run_with_context`
3342    /// path (which would fail earlier on worktree setup in a unit-test environment).
3343    #[tokio::test]
3344    async fn update_agent_label_returns_false_on_set_labels_failure() {
3345        let task = ExternalTask {
3346            id: ExternalId("test-reroute-1".to_string()),
3347            title: "Test".to_string(),
3348            body: "body".to_string(),
3349            state: "open".to_string(),
3350            labels: vec![],
3351            author: "bot".to_string(),
3352            created_at: "2026-01-01T00:00:00Z".to_string(),
3353            updated_at: "2026-01-01T00:00:00Z".to_string(),
3354            url: String::new(),
3355        };
3356        let backend: Arc<dyn ExternalBackend> =
3357            Arc::new(LabelFailingBackend::new().with_set_labels_err("network error"));
3358
3359        let result =
3360            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
3361                .await;
3362
3363        assert!(
3364            !result,
3365            "update_agent_label_after_reroute should return false when set_labels fails"
3366        );
3367    }
3368
3369    /// `update_agent_label_after_reroute` returns `true` when `set_labels` succeeds.
3370    #[tokio::test]
3371    async fn update_agent_label_returns_true_on_success() {
3372        let task = ExternalTask {
3373            id: ExternalId("test-reroute-2".to_string()),
3374            title: "Test".to_string(),
3375            body: "body".to_string(),
3376            state: "open".to_string(),
3377            labels: vec![],
3378            author: "bot".to_string(),
3379            created_at: "2026-01-01T00:00:00Z".to_string(),
3380            updated_at: "2026-01-01T00:00:00Z".to_string(),
3381            url: String::new(),
3382        };
3383        // LabelFailingBackend with no injected error delegates to TrackingBackend (always Ok).
3384        let backend: Arc<dyn ExternalBackend> = Arc::new(LabelFailingBackend::new());
3385
3386        let result =
3387            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
3388                .await;
3389
3390        // ensure_label is skipped in #[cfg(test)], set_labels succeeds via mock.
3391        assert!(
3392            result,
3393            "update_agent_label_after_reroute should return true when set_labels succeeds"
3394        );
3395    }
3396
3397    // ── NDJSON envelope fallback (issue #3078) ────────────────────────────────
3398
3399    /// Pattern A: agent emits domain JSON as result (no `status` field).
3400    /// NDJSON envelope says is_error=false, subtype=success.
3401    /// Was previously returning InvalidResponse → task re-ran.
3402    #[test]
3403    fn parse_success_output_ndjson_envelope_fallback_domain_json_result() {
3404        let domain_result = r#"{"file_path":"md/journal/2026-05-08-morning-briefing.md","summary":"Morning briefing filed","health":{"status":"ok"},"priorities":[]}"#;
3405        let ndjson = format!(
3406            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{domain_result:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
3407        );
3408
3409        let runner = agents::get_runner("glm");
3410        let parsed = parse_success_output("149200", "glm", &*runner, &ndjson)
3411            .expect("should synthesize done from NDJSON envelope when result is domain JSON");
3412
3413        assert_eq!(parsed.response.status, "done");
3414        assert!(
3415            parsed.response.summary.contains("file_path") || !parsed.response.summary.is_empty(),
3416            "summary should be populated from domain JSON result text"
3417        );
3418        assert_eq!(parsed.input_tokens, Some(3000));
3419        assert_eq!(parsed.output_tokens, Some(400));
3420    }
3421
3422    /// Grok-style envelope: no terminal_reason, subtype=success.
3423    #[test]
3424    fn parse_success_output_envelope_fallback_without_terminal_reason() {
3425        let receipt = r#"{"ok":true,"report":"notes.md","commit":"abc"}"#;
3426        let ndjson = format!(
3427            r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","result":{receipt:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
3428        );
3429        let runner = agents::get_runner("grok");
3430        let parsed = parse_success_output("1", "grok", &*runner, &ndjson).expect("done");
3431        assert_eq!(parsed.response.status, "done");
3432        assert_eq!(parsed.input_tokens, Some(3000));
3433    }
3434
3435    /// Pattern B: agent emits plain prose as result, no completion keywords.
3436    /// NDJSON envelope says is_error=false, subtype=success.
3437    /// Was previously returning InvalidResponse → task re-ran.
3438    #[test]
3439    fn parse_success_output_ndjson_envelope_fallback_plain_prose_result() {
3440        let prose = "PR #1008 now has zero diff vs main. The reverted ledger changes are gone and the branch is clean.";
3441        let ndjson = format!(
3442            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{prose:?},"usage":{{"input_tokens":2500,"output_tokens":300}}}}"#
3443        );
3444
3445        let runner = agents::get_runner("claude");
3446        let parsed = parse_success_output("149164", "claude", &*runner, &ndjson)
3447            .expect("should synthesize done from NDJSON envelope when result is plain prose");
3448
3449        assert_eq!(parsed.response.status, "done");
3450        assert_eq!(parsed.response.summary, prose);
3451        assert_eq!(parsed.input_tokens, Some(2500));
3452        assert_eq!(parsed.output_tokens, Some(300));
3453    }
3454
3455    /// Verify that is_error=true in NDJSON envelope still returns AgentFailed,
3456    /// not "done" — the envelope fallback must not override explicit errors.
3457    #[test]
3458    fn parse_success_output_ndjson_envelope_fallback_skipped_when_is_error_true() {
3459        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"result":"Something went wrong with no status"}"#;
3460
3461        let runner = agents::get_runner("claude");
3462        let err = parse_success_output("149999", "claude", &*runner, ndjson)
3463            .expect_err("is_error=true should return AgentFailed");
3464
3465        assert!(matches!(err, agents::AgentError::AgentFailed { .. }));
3466    }
3467
3468    fn tool_pair(id: &str, name: &str, cmd: &str, is_error: bool) -> String {
3469        format!(
3470            "{}\n{}\n",
3471            serde_json::json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":id,"name":name,"input":{"command":cmd}}]}}),
3472            serde_json::json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":id,"is_error":is_error,"content":"x"}]}}),
3473        )
3474    }
3475
3476    #[test]
3477    fn salvage_wrapup_accepts_commit_then_cleanup() {
3478        let raw = format!(
3479            "{}{}",
3480            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3481            tool_pair("2", "run_terminal_cmd", "git status", false)
3482        );
3483        assert!(salvage_wrapup(&raw).is_some());
3484    }
3485
3486    #[test]
3487    fn salvage_wrapup_rejects_edit_after_commit_or_unanswered_call() {
3488        let edit = format!(
3489            "{}{}",
3490            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3491            tool_pair("2", "search_replace", "", false)
3492        );
3493        assert!(salvage_wrapup(&edit).is_none());
3494        let pending = format!(
3495            "{}{}",
3496            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3497            serde_json::json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":"2","name":"run_terminal_cmd","input":{"command":"cargo test"}}]}})
3498        );
3499        assert!(salvage_wrapup(&pending).is_none());
3500        assert!(salvage_wrapup(&tool_pair("1", "run_terminal_cmd", "ls", false)).is_none());
3501    }
3502
3503    // ── terminal_reason:completed + exit 1 detection (issue #3087) ─────────────
3504
3505    /// kimi/minimax sometimes emit valid NDJSON with terminal_reason=completed
3506    /// but exit 1. In that case parse_success_output may fail (e.g. empty result
3507    /// text that synthesize can't parse), then parse_session_output would
3508    /// normally fall back to classify_error — grabbing a garbled JSON tail and
3509    /// producing a spurious Unknown/Auth error. The fix intercepts this and
3510    /// returns InvalidResponse (soft error, re-route) instead.
3511    #[test]
3512    fn parse_session_output_exit0_without_envelope_is_interrupted() {
3513        let session = session::SessionOutput {
3514            exit_code: 0,
3515            raw_stdout: "\u{4}\u{4}\u{4} truncated pane junk".to_string(),
3516            raw_stderr: String::new(),
3517            elapsed_secs: Some(180),
3518            sandbox_error: None,
3519        };
3520
3521        let runner = agents::get_runner("claude");
3522        let err = parse_session_output("3876", "claude", &*runner, &session)
3523            .expect_err("missing envelope should be an error");
3524        assert!(
3525            matches!(err, agents::AgentError::Interrupted { .. }),
3526            "exit 0 with no result envelope should be Interrupted, got {err:?}"
3527        );
3528    }
3529
3530    #[test]
3531    fn parse_session_output_exit1_with_terminal_reason_completed_returns_invalid_response() {
3532        // Empty result text causes synthesize_response_from_text to return None
3533        // (empty/whitespace-only input), so parse_success_output returns
3534        // InvalidResponse. With the fix, parse_session_output intercepts the
3535        // "terminal_reason":"completed" signal before classify_error and
3536        // returns InvalidResponse (re-route) instead of a garbled error.
3537        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":"","usage":{"input_tokens":1000,"output_tokens":100}}"#;
3538        let session = session::SessionOutput {
3539            exit_code: 1,
3540            raw_stdout: ndjson.to_string(),
3541            raw_stderr: String::new(),
3542            elapsed_secs: Some(30),
3543            sandbox_error: None,
3544        };
3545
3546        let runner = agents::get_runner("claude");
3547        let result = parse_session_output("149298", "claude", &*runner, &session);
3548
3549        let err = result.expect_err("should be an error, not Ok");
3550        assert!(
3551            matches!(err, agents::AgentError::InvalidResponse { .. }),
3552            "should be InvalidResponse, got {err:?}"
3553        );
3554    }
3555
3556    // ── GLM cost-telemetry exit-1 detection (issue #3094) ──────────────────────
3557
3558    /// GLM exits with code 1 but stdout is only cost telemetry JSON (contains
3559    /// "costUSD":). This must return InvalidResponse (soft re-route) instead of
3560    /// a hard Failed classification with spurious agent cooldown.
3561    #[test]
3562    fn parse_session_output_exit1_with_cost_telemetry_returns_invalid_response() {
3563        // GLM emits cost/usage JSON on clean exit; exit code 1 is a false failure.
3564        let telemetry = r#"{"type":"result","subtype":"success","is_error":false,"result":"","costUSD":0.042,"usage":{"input_tokens":5000,"output_tokens":800}}"#;
3565        let session = session::SessionOutput {
3566            exit_code: 1,
3567            raw_stdout: telemetry.to_string(),
3568            raw_stderr: String::new(),
3569            elapsed_secs: Some(120),
3570            sandbox_error: None,
3571        };
3572
3573        let runner = agents::get_runner("glm");
3574        let result = parse_session_output("149347", "glm", &*runner, &session);
3575
3576        let err = result.expect_err("should be an error, not Ok");
3577        assert!(
3578            matches!(err, agents::AgentError::InvalidResponse { .. }),
3579            "glm cost-telemetry exit-1 should return InvalidResponse, got {err:?}"
3580        );
3581    }
3582
3583    /// GLM with costUSD AND is_error:true — must propagate AgentFailed, not
3584    /// silently swallow the error via the cost-telemetry guard.
3585    #[test]
3586    fn parse_session_output_exit1_cost_telemetry_with_is_error_true_propagates_failure() {
3587        let telemetry = r#"{"type":"result","subtype":"error","is_error":true,"result":"tool failed","costUSD":0.01,"usage":{"input_tokens":100,"output_tokens":10}}"#;
3588        let session = session::SessionOutput {
3589            exit_code: 1,
3590            raw_stdout: telemetry.to_string(),
3591            raw_stderr: String::new(),
3592            elapsed_secs: Some(15),
3593            sandbox_error: None,
3594        };
3595
3596        let runner = agents::get_runner("glm");
3597        let result = parse_session_output("149348", "glm", &*runner, &session);
3598
3599        let err = result.expect_err("should be an error");
3600        assert!(
3601            matches!(err, agents::AgentError::AgentFailed { .. }),
3602            "is_error=true with costUSD must still return AgentFailed, got {err:?}"
3603        );
3604    }
3605
3606    /// terminal_reason=completed must NOT override actual error conditions.
3607    /// When is_error=true in the NDJSON envelope, AgentFailed must be returned
3608    /// even if terminal_reason=completed is also present. Only kimi/minimax
3609    /// agents use find_claude_result which extracts is_error, so use "kimi"
3610    /// as agent_name.
3611    #[test]
3612    fn parse_session_output_exit1_preserves_error_when_is_error_true() {
3613        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"terminal_reason":"completed","result":"failed"}"#;
3614        let session = session::SessionOutput {
3615            exit_code: 1,
3616            raw_stdout: ndjson.to_string(),
3617            raw_stderr: String::new(),
3618            elapsed_secs: Some(30),
3619            sandbox_error: None,
3620        };
3621
3622        let runner = agents::get_runner("kimi");
3623        let result = parse_session_output("149299", "kimi", &*runner, &session);
3624
3625        let err = result.expect_err("should be an error");
3626        assert!(
3627            matches!(err, agents::AgentError::AgentFailed { .. }),
3628            "is_error=true should return AgentFailed, got {err:?}"
3629        );
3630    }
3631
3632    /// RateLimit with terminal_reason=completed must be propagated as RateLimit,
3633    /// not downgraded to InvalidResponse. This covers claude session-limit errors
3634    /// that include "terminal_reason":"completed" in their NDJSON envelope — the
3635    /// terminal_reason guard was designed for kimi/minimax/glm false-positive
3636    /// exit codes but was intercepting classified errors like RateLimit.
3637    #[test]
3638    fn parse_session_output_exit1_with_rate_limit_and_terminal_reason_completed() {
3639        let ndjson = r#"{"type":"result","is_error":true,"terminal_reason":"completed","result":"You've hit your session limit · resets 2:20pm (America/Sao_Paulo)","usage":{"input_tokens":0,"output_tokens":0}}"#;
3640        let session = session::SessionOutput {
3641            exit_code: 1,
3642            raw_stdout: ndjson.to_string(),
3643            raw_stderr: String::new(),
3644            elapsed_secs: Some(120),
3645            sandbox_error: None,
3646        };
3647
3648        let runner = agents::get_runner("claude");
3649        let result = parse_session_output("152895", "claude", &*runner, &session);
3650
3651        let err = result.expect_err("rate-limit should be an error");
3652        assert!(
3653            matches!(err, agents::AgentError::RateLimit { .. }),
3654            "session limit should return RateLimit, not downgraded, got {err:?}"
3655        );
3656    }
3657}