Skip to main content

orch/engine/runner/
mod.rs

1//! Task runner — executes tasks using AI agents.
2//!
3//! Replaces `run_task.sh` with pure Rust. The runner:
4//! 1. Sets up a git worktree for isolation
5//! 2. Builds context (project instructions, comments)
6//! 3. Renders prompts and spawns the agent in tmux
7//! 4. Waits for completion, collects and classifies the response
8//! 5. Handles errors (reroute on limits, fallback agents)
9//! 6. Auto-commits, pushes, and creates PRs
10//! 7. Updates status labels and posts result comments
11//!
12//! Submodules handle the heavy lifting:
13//! - [`task_init`] — guard checks, worktree setup, and invocation building
14//! - [`session`] — tmux session lifecycle and output collection
15//! - [`response_handler`] — success path: commit, push, PR, token storage
16//! - [`fallback`] — error classification and recovery strategies
17
18pub mod agent;
19pub mod agents;
20pub mod commit;
21pub mod context;
22pub mod cow;
23pub mod diff;
24pub mod direct;
25pub mod fallback;
26pub mod git_ops;
27#[cfg(test)]
28mod recorded_fixtures;
29pub mod response;
30pub mod response_handler;
31pub mod sandbox;
32pub mod screenshots;
33pub mod session;
34pub mod task_init;
35pub mod worktree;
36
37use crate::backends::{ExternalBackend, ExternalTask, Status};
38use crate::config;
39use crate::engine;
40use crate::engine::cleanup;
41use crate::engine::cooldown;
42use crate::engine::router::RouteResult;
43use crate::engine::tasks;
44use crate::engine::tasks::is_internal_id;
45#[cfg(not(test))]
46use crate::github::http;
47use crate::github::http::GhHttp;
48use crate::home;
49use crate::parser;
50use crate::parser::AgentResponse;
51use crate::parser::Delegation;
52use crate::status_heuristics;
53use crate::store;
54use crate::store::InsertTaskMetric;
55use crate::store::RunTokenUsage;
56use crate::store::TaskStatus;
57use crate::store::TaskStore;
58use crate::tmux::TmuxManager;
59use anyhow::Context;
60use chrono::Utc;
61pub use response::WeightSignal;
62use serde_json::Value;
63use std::collections::HashSet;
64use std::env;
65use std::error;
66use std::fmt;
67use std::fmt::Debug;
68use std::fmt::Formatter;
69use std::path::PathBuf;
70use std::slice;
71use std::sync::Arc;
72use std::time::Duration;
73use tokio::fs;
74use tokio::task;
75
76/// Fully materialized task-run audit data.
77#[derive(Debug, Clone)]
78pub struct RunAudit {
79    pub stdout: String,
80    pub stderr: String,
81    pub parsed_response: String,
82    pub outcome: String,
83    pub error: String,
84    pub input_tokens: u64,
85    pub output_tokens: u64,
86    pub total_cost_usd: f64,
87    pub duration_secs: f64,
88}
89
90/// Result returned by `run()` so callers can persist the audit trail.
91#[derive(Debug, Clone)]
92pub struct RunExecution {
93    pub status: String,
94    pub exit_code: Option<i32>,
95    pub audit: RunAudit,
96}
97
98fn serialize_parsed_response(
99    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
100    raw_stdout: &str,
101) -> String {
102    match parse_result {
103        Ok(parsed) => serde_json::json!({
104            "response": parsed.response,
105            "input_tokens": parsed.input_tokens,
106            "output_tokens": parsed.output_tokens,
107            "cost_usd": parsed.cost_usd,
108            "duration_ms": parsed.duration_ms,
109        })
110        .to_string(),
111        Err(agents::AgentError::InvalidResponse { raw }) => {
112            serde_json::json!({"raw": raw}).to_string()
113        }
114        Err(_) => serde_json::json!({"raw": raw_stdout}).to_string(),
115    }
116}
117
118fn classify_run_outcome(
119    status: &str,
120    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
121    push_failed: bool,
122) -> &'static str {
123    if push_failed {
124        return "push_failed";
125    }
126    match parse_result {
127        Err(agents::AgentError::Timeout { .. }) => "timeout",
128        Err(agents::AgentError::Interrupted { .. }) => "interrupted",
129        // Generic rate-limit patterns (e.g. "usage limit") also match billing-cycle
130        // exhaustion messages, so consult the credit-exhaustion detector first —
131        // otherwise Kimi-style "usage limit for this billing cycle" errors are
132        // stored as a plain "rate_limit" outcome (issue #3529).
133        Err(agents::AgentError::RateLimit { message }) => {
134            cooldown::detect_credit_exhaustion_outcome(message).unwrap_or("rate_limit")
135        }
136        Err(agents::AgentError::Auth { message }) => {
137            cooldown::detect_credit_exhaustion_outcome(message).unwrap_or("failed")
138        }
139        Err(agents::AgentError::InvalidResponse { .. }) => "parse_error",
140        Ok(_)
141            if matches!(
142                status,
143                "done" | "completed" | "in_progress" | "in_review" | "needs_review" | "routed"
144            ) =>
145        {
146            "success"
147        }
148        Ok(_) if status == "blocked" => "blocked",
149        // Non-canonical status that clearly reads as completion — success, not
150        // "unrecognized status" (#3627).
151        Ok(_) if status_heuristics::status_looks_like_descriptive_completion(status) => "success",
152        Err(_) | Ok(_) => "failed",
153    }
154}
155
156fn classify_run_error_type(last_error: &str) -> &'static str {
157    if last_error.is_empty() {
158        // No error: agent completed successfully and created a PR waiting for review
159        "success"
160    } else if last_error.contains("timeout") {
161        "timeout"
162    } else if last_error.contains("interrupted") {
163        "interrupted"
164    } else if last_error.contains("billing cycle") || last_error.contains("monthly quota") {
165        "billing_cycle_exhausted"
166    } else if last_error.contains("rate limit") || last_error.contains("usage limit") {
167        "rate_limit"
168    } else if last_error.contains("auth") {
169        "auth_error"
170    } else if last_error.contains("push failed") {
171        "push_failed"
172    } else if last_error.contains("No commits between") || last_error.contains("no commits") {
173        "no_commits"
174    } else if last_error.contains("create PR failed")
175        || last_error.contains("failed to create pull request")
176        || last_error.contains("pull request creation failed")
177    {
178        "pr_failed"
179    } else if last_error.contains("invalid response") || last_error.contains("parse error") {
180        "parse_error"
181    } else if last_error.contains("exceeded max attempts") {
182        "max_attempts"
183    } else {
184        let lower = last_error.to_ascii_lowercase();
185        if lower.contains("build failed")
186            || lower.contains("tests failed")
187            || lower.contains("test failed")
188            || lower.contains("test suite failed")
189            || lower.contains("lint failed")
190        {
191            "ci_failure"
192        } else {
193            "failed"
194        }
195    }
196}
197
198fn extract_run_tokens(
199    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
200) -> (u64, u64) {
201    match parse_result {
202        Ok(parsed) => (
203            parsed
204                .input_tokens
205                .or(parsed.response.input_tokens)
206                .unwrap_or(0),
207            parsed
208                .output_tokens
209                .or(parsed.response.output_tokens)
210                .unwrap_or(0),
211        ),
212        Err(_) => (0, 0),
213    }
214}
215
216#[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
217fn run_duration_secs(started_at: &chrono::DateTime<Utc>) -> f64 {
218    (Utc::now() - *started_at).num_milliseconds() as f64 / 1000.0
219}
220
221struct RunAuditInput<'a> {
222    task_id: &'a str,
223    status: &'a str,
224    parse_result: &'a Result<agents::ParsedResponse, agents::AgentError>,
225    raw_stdout: &'a str,
226    raw_stderr: &'a str,
227    started_at: &'a chrono::DateTime<Utc>,
228    error_override: Option<String>,
229    elapsed_secs: Option<u64>,
230    push_failed: bool,
231}
232
233fn parse_success_output(
234    task_id: &str,
235    agent_name: &str,
236    agent_runner: &dyn agents::AgentRunner,
237    raw_stdout: &str,
238) -> Result<agents::ParsedResponse, agents::AgentError> {
239    // The last result envelope is the agent's final answer and carries the run's tokens and cost.
240    // Claude writes terminal_reason:completed on every run, so this goes before that shortcut
241    let agent_result = agents::find_agent_result(agent_name, raw_stdout);
242    if let Some(result) = &agent_result
243        && !result.is_error
244        && let Ok(response) = parser::parse(&result.result_text)
245    {
246        return Ok(agents::ParsedResponse {
247            response,
248            input_tokens: result.input_tokens,
249            output_tokens: result.output_tokens,
250            cost_usd: result.cost_usd,
251            duration_ms: result.duration_ms,
252        });
253    }
254
255    // Quick NDJSON terminal_reason:completed shortcut: when the agent
256    // emitted a session envelope that explicitly signals completion we
257    // prefer to interpret that as a successful run (or at least a
258    // soft/empty success) rather than immediately classifying an error
259    // from the trailing process output.  Do not override explicit
260    // in-envelope `is_error: true` signals.
261    if raw_stdout.contains("\"terminal_reason\":\"completed\"")
262        && !raw_stdout.contains("\"is_error\":true")
263    {
264        // Try to recover a structured response from assistant messages first
265        let assistant_text = agents::collect_assistant_messages_text(agent_name, raw_stdout);
266        if !assistant_text.is_empty() {
267            if let Ok(response) = parser::parse(&assistant_text) {
268                tracing::debug!(
269                    task_id,
270                    agent = agent_name,
271                    "found AgentResponse JSON in earlier assistant message (terminal_reason:completed)"
272                );
273                return Ok(agents::ParsedResponse {
274                    response,
275                    input_tokens: None,
276                    output_tokens: None,
277                    cost_usd: None,
278                    duration_ms: None,
279                });
280            }
281            if let Some(response) = agents::synthesize_response_from_text(&assistant_text) {
282                tracing::warn!(
283                    task_id,
284                    agent = agent_name,
285                    "synthesizing response from assistant message due to terminal_reason:completed"
286                );
287                return Ok(agents::ParsedResponse {
288                    response,
289                    input_tokens: None,
290                    output_tokens: None,
291                    cost_usd: None,
292                    duration_ms: None,
293                });
294            }
295        }
296        // If we couldn't recover structured data from earlier assistant
297        // messages, fall through and let the regular NDJSON envelope parsing
298        // / agent-specific parsers handle the terminal_reason case. This
299        // preserves the existing behavior where an empty `result` field
300        // combined with exit != 0 becomes an InvalidResponse that is then
301        // interpreted by parse_session_output (see tests).
302    }
303
304    // 1. Try to use the unified agent result extractor first
305    if let Some(agent_result) = agent_result {
306        // Treat is_error as authoritative for the success path
307        if agent_result.is_error {
308            // Classify the error from the agent's result text rather than
309            // blindly returning AgentFailed — this preserves rate-limit, auth,
310            // and other actionable error types for proper cooldown/reroute.
311            // Without this, e.g. a claude session-limit error with is_error=true
312            // would be downgraded to AgentFailed, losing the RateLimit signal
313            // that the fallback chain needs for UsageLimit recovery (issue #3292).
314            if let Some(e) = agents::patterns::detect_rate_limit(&agent_result.result_text) {
315                return Err(e);
316            }
317            if let Some(e) = agents::patterns::detect_auth_error(&agent_result.result_text) {
318                return Err(e);
319            }
320            if let Some(e) = agents::patterns::detect_model_unavailable(&agent_result.result_text) {
321                return Err(e);
322            }
323            // Grok puts "the wait for background work ran out" in errors[] and
324            // exits 1. That is a timeout, so handle_failover retries
325            let classified = agent_runner.classify_error(1, raw_stdout, "");
326            if matches!(classified, agents::AgentError::Timeout { .. }) {
327                return Err(classified);
328            }
329            return Err(agents::AgentError::AgentFailed {
330                message: agent_result.result_text,
331            });
332        }
333
334        // Strategy 2: scan all earlier assistant messages in the NDJSON
335        // stream for valid AgentResponse JSON. Agents often emit the
336        // JSON in an intermediate turn and then summarise in prose as
337        // their final output, so the `type:result` text may be pure
338        // narrative while a prior assistant turn contains the JSON.
339        let assistant_text = agents::collect_assistant_messages_text(agent_name, raw_stdout);
340        if !assistant_text.is_empty()
341            && let Ok(response) = parser::parse(&assistant_text)
342        {
343            tracing::debug!(
344                task_id,
345                agent = agent_name,
346                "found AgentResponse JSON in earlier assistant message"
347            );
348            return Ok(agents::ParsedResponse {
349                response,
350                input_tokens: agent_result.input_tokens,
351                output_tokens: agent_result.output_tokens,
352                cost_usd: agent_result.cost_usd,
353                duration_ms: agent_result.duration_ms,
354            });
355        }
356
357        // Only fall back to text synthesis when parsing fails
358        if let Some(response) = agents::synthesize_response_from_text(&agent_result.result_text) {
359            tracing::warn!(
360                task_id,
361                agent = agent_name,
362                "parse failed on agent result, synthesizing response from plain text"
363            );
364            return Ok(agents::ParsedResponse {
365                response,
366                input_tokens: agent_result.input_tokens,
367                output_tokens: agent_result.output_tokens,
368                cost_usd: agent_result.cost_usd,
369                duration_ms: agent_result.duration_ms,
370            });
371        }
372
373        // NDJSON envelope fallback: the envelope explicitly reports
374        // success (is_error=false) but the result content is domain
375        // JSON or plain prose that doesn't match AgentResponse schema
376        // and doesn't contain completion keywords.  Rather than
377        // returning InvalidResponse (which causes the task to re-run),
378        // synthesize a minimal "done" response from the result text.
379        //
380        // Guard against kimi/minimax edge case: when terminal_reason=completed
381        // but result text is empty, synthesize returns None, parse_success_output
382        // returns InvalidResponse, and parse_session_output can intercept it
383        // before classify_error. Requiring non-empty result here means the
384        // envelope fallback only fires when we have actual text to report.
385        if !agent_result.is_error
386            && result_envelope_completed(raw_stdout)
387            && !agent_result.result_text.is_empty()
388        {
389            let summary_end = agents::truncate_at_char_boundary(&agent_result.result_text, 500);
390            let summary = agent_result.result_text[..summary_end].to_string();
391            tracing::warn!(
392                task_id,
393                agent = agent_name,
394                "NDJSON envelope indicates success but result lacks AgentResponse \
395                 schema — synthesizing done response"
396            );
397            return Ok(agents::ParsedResponse {
398                response: AgentResponse {
399                    status: "done".to_string(),
400                    summary,
401                    ..Default::default()
402                },
403                input_tokens: agent_result.input_tokens,
404                output_tokens: agent_result.output_tokens,
405                cost_usd: agent_result.cost_usd,
406                duration_ms: agent_result.duration_ms,
407            });
408        }
409    }
410
411    // 2. Fall back to generic agent-specific parser if extraction failed
412    match agent_runner.parse_response(raw_stdout) {
413        Ok(parsed) => Ok(parsed),
414        Err(agents::AgentError::InvalidResponse { raw }) => {
415            if let Some(response) = agents::synthesize_response_from_text(&raw) {
416                tracing::warn!(
417                    task_id,
418                    agent = agent_name,
419                    "parse failed, synthesizing response from plain text"
420                );
421                // Try to extract token metadata from the original raw NDJSON
422                // output (not the truncated/extracted text from the error), so
423                // cost tracking isn't lost for agents that return free-form text.
424                let agent_result = agents::find_agent_result(agent_name, raw_stdout);
425                let input_tokens = agent_result.as_ref().and_then(|r| r.input_tokens);
426                let output_tokens = agent_result.as_ref().and_then(|r| r.output_tokens);
427                let cost_usd = agent_result.as_ref().and_then(|r| r.cost_usd);
428                let duration_ms = agent_result.as_ref().and_then(|r| r.duration_ms);
429                Ok(agents::ParsedResponse {
430                    response,
431                    input_tokens,
432                    output_tokens,
433                    cost_usd,
434                    duration_ms,
435                })
436            } else {
437                Err(agents::AgentError::InvalidResponse { raw })
438            }
439        }
440        Err(err) => Err(err),
441    }
442}
443
444/// Whether the last `type: result` line reports `subtype: success` or
445/// `terminal_reason: completed`. Grok omits `terminal_reason`.
446fn result_envelope_line(raw_stdout: &str) -> Option<serde_json::Value> {
447    raw_stdout
448        .lines()
449        .rev()
450        .filter_map(|l| serde_json::from_str::<serde_json::Value>(l.trim()).ok())
451        .find(|v| v.get("type").and_then(|t| t.as_str()) == Some("result"))
452}
453
454fn has_result_envelope(raw_stdout: &str) -> bool {
455    result_envelope_line(raw_stdout).is_some()
456}
457
458fn result_envelope_completed(raw_stdout: &str) -> bool {
459    result_envelope_line(raw_stdout).is_some_and(|v| {
460        v.get("subtype").and_then(|t| t.as_str()) == Some("success")
461            || v.get("terminal_reason").and_then(|t| t.as_str()) == Some("completed")
462    })
463}
464
465// A timeout kill has no result envelope. Salvage only the wrap-up shape: a successful
466// `git commit`, then only shell calls that all returned ok. Returns the last assistant text
467fn salvage_wrapup(raw: &str) -> Option<String> {
468    // (tool name, command, finished ok) in call order
469    let mut calls: Vec<(String, String, Option<bool>)> = Vec::new();
470    let mut last_text = String::new();
471    for v in raw
472        .lines()
473        .filter_map(|l| serde_json::from_str::<Value>(l.trim()).ok())
474    {
475        let blocks = v.pointer("/message/content").and_then(|c| c.as_array());
476        for b in blocks.into_iter().flatten() {
477            match b.get("type").and_then(|t| t.as_str()) {
478                Some("tool_use") => calls.push((
479                    b["name"].as_str().unwrap_or("").to_string(),
480                    b.pointer("/input/command")
481                        .and_then(|c| c.as_str())
482                        .unwrap_or("")
483                        .to_string(),
484                    None,
485                )),
486                Some("text") if v["type"] == "assistant" => {
487                    last_text = b["text"].as_str().unwrap_or("").to_string();
488                }
489                Some("tool_result") => {
490                    let ok = b.get("is_error").and_then(serde_json::Value::as_bool) != Some(true);
491                    // results arrive in call order, fill the first unanswered call
492                    if let Some(c) = calls.iter_mut().find(|c| c.2.is_none()) {
493                        c.2 = Some(ok);
494                    }
495                }
496                _ => {}
497            }
498        }
499    }
500    let commit = calls
501        .iter()
502        .rposition(|(_, cmd, ok)| cmd.contains("git commit") && *ok == Some(true))?;
503    let tail_ok = calls[commit..].iter().all(|(name, _, ok)| {
504        matches!(name.as_str(), "Bash" | "run_terminal_cmd") && *ok == Some(true)
505    });
506    let lower = last_text.to_lowercase();
507    if !tail_ok || lower.contains("still running") || lower.contains("in progress") {
508        return None;
509    }
510    Some(last_text.chars().take(500).collect())
511}
512
513/**
514 * Tries `parse_success_output` first when stdout is non-empty. If that succeeds,
515 * returns success regardless of exit code. If it fails and `exit_code != 0`,
516 * checks for `"terminal_reason":"completed"` or cost-telemetry-only stdout
517 * (`"costUSD":`) before falling back to `classify_error`, so valid sessions that
518 * exit 1 produce `InvalidResponse` (soft error, re-route) instead of a garbled
519 * `Unknown`/`Auth` error (hard failure with spurious cooldown)
520 **/
521pub fn parse_session_output(
522    task_id: &str,
523    agent_name: &str,
524    agent_runner: &dyn agents::AgentRunner,
525    session_output: &session::SessionOutput,
526) -> Result<agents::ParsedResponse, agents::AgentError> {
527    // Some agents may emit NDJSON/telemetry to stderr instead of stdout. Combine
528    // both streams so parse_success_output can inspect either place for the
529    // terminal_reason/cost telemetry envelope. Only skip parsing when both are
530    // empty so we still attempt recovery when stderr contains the envelope.
531    let combined_output = format!("{}{}", session_output.raw_stdout, session_output.raw_stderr);
532    if !(session_output.raw_stdout.is_empty() && session_output.raw_stderr.is_empty()) {
533        match parse_success_output(task_id, agent_name, agent_runner, &combined_output) {
534            Ok(parsed) => return Ok(parsed),
535            Err(err) => {
536                if session_output.exit_code == 124 {
537                    if let Some(summary) = salvage_wrapup(&combined_output) {
538                        tracing::warn!(
539                            task_id,
540                            agent = agent_name,
541                            "timed out after commit during wrap-up, salvaging as done"
542                        );
543                        return Ok(agents::ParsedResponse {
544                            response: AgentResponse {
545                                status: "done".to_string(),
546                                summary,
547                                ..Default::default()
548                            },
549                            input_tokens: None,
550                            output_tokens: None,
551                            cost_usd: None,
552                            duration_ms: None,
553                        });
554                    }
555                    // timeout(1) killed the process. A cancelled error envelope is the CLI
556                    // reacting to SIGTERM
557                    return Err(agent_runner.classify_error_with_elapsed(
558                        session_output.exit_code,
559                        &session_output.raw_stdout,
560                        &session_output.raw_stderr,
561                        session_output.elapsed_secs,
562                    ));
563                }
564                // If parse_success_output already returned a classified error
565                // (RateLimit, AgentFailed, Timeout, Auth, etc.), propagate it
566                // directly — only InvalidResponse (unparseable output) should
567                // be intercepted by the terminal_reason:completed / cost-telemetry
568                // guard below. Without this, e.g. a claude session-limit error
569                // with terminal_reason=completed would be downgraded from RateLimit
570                // to InvalidResponse → Failed, losing the correct cooldown/reroute.
571                if !matches!(err, agents::AgentError::InvalidResponse { .. }) {
572                    let mut err = err;
573                    if let agents::AgentError::Timeout { elapsed } = &mut err
574                        && let Some(secs) = session_output.elapsed_secs
575                    {
576                        *elapsed = Duration::from_secs(secs);
577                    }
578                    return Err(err);
579                }
580                if session_output.exit_code != 0 {
581                    // kimi/minimax/glm sometimes emit valid NDJSON with
582                    // "terminal_reason":"completed" or cost-only telemetry
583                    // ("costUSD":) but exit 1 due to a post-session hook or
584                    // minor CLI issue. Scanning the tail would just grab JSON
585                    // metadata, producing a garbled error. Detect either
586                    // completion signal and treat as a soft parse error so the
587                    // task re-routes instead of hitting a false cooldown from
588                    // Unknown or Auth error variants. Use the combined output
589                    // (stdout+stderr) so envelopes written to stderr are
590                    // observed.
591                    let combined = &combined_output;
592                    let is_completed_signal =
593                        combined.contains("\"terminal_reason\":\"completed\"");
594                    // GLM emits a cost-telemetry JSON object (containing
595                    // "costUSD":) as its final stdout/stderr line on clean
596                    // exit even when the process exits with code 1. If the
597                    // only content is such telemetry (no is_error=true in the
598                    // envelope), treat it the same as terminal_reason:completed.
599                    let is_cost_telemetry_only = combined.contains("\"costUSD\":")
600                        && !combined.contains("\"is_error\":true");
601                    if is_completed_signal || is_cost_telemetry_only {
602                        let raw_tail = agents::patterns::safe_tail(combined, 300);
603                        return Err(agents::AgentError::InvalidResponse {
604                            raw: raw_tail.to_string(),
605                        });
606                    }
607                    return Err(agent_runner.classify_error_with_elapsed(
608                        session_output.exit_code,
609                        &session_output.raw_stdout,
610                        &session_output.raw_stderr,
611                        session_output.elapsed_secs,
612                    ));
613                }
614                if !has_result_envelope(&combined_output) {
615                    return Err(agents::AgentError::Interrupted {
616                        message: "exited without a result envelope".to_string(),
617                    });
618                }
619                return Err(err);
620            }
621        }
622    }
623
624    if session_output.exit_code != 0 {
625        return Err(agent_runner.classify_error_with_elapsed(
626            session_output.exit_code,
627            &session_output.raw_stdout,
628            &session_output.raw_stderr,
629            session_output.elapsed_secs,
630        ));
631    }
632
633    Err(agents::AgentError::Unknown {
634        exit_code: session_output.exit_code,
635        message: "empty-output-exit0: opencode returned exit 0 with empty stdout".to_string(),
636    })
637}
638
639/// The repo has no project directory: not in the active config, no legacy
640/// `project_dir`, no bare clone. It fails before any agent starts
641#[derive(Debug)]
642pub(crate) struct NoProjectDir {
643    pub(crate) repo: String,
644    pub(crate) bare_clone: PathBuf,
645}
646
647impl NoProjectDir {
648    /// Block reason for a task of this repo
649    pub(crate) const BLOCK_REASON: &str = "project is not configured and has no bare clone";
650}
651
652impl fmt::Display for NoProjectDir {
653    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
654        write!(
655            f,
656            "no resolvable project directory for repo {} — not in active config, no bare clone at {}, no legacy project_dir set",
657            self.repo,
658            self.bare_clone.display()
659        )
660    }
661}
662
663impl error::Error for NoProjectDir {}
664
665/// Task runner configuration.
666pub struct TaskRunner {
667    /// Repository slug (owner/repo)
668    repo: String,
669    /// Path to the orch home directory
670    orch_home: PathBuf,
671    /// Unified task store for run audit trail
672    store: Option<Arc<TaskStore>>,
673}
674
675impl Debug for TaskRunner {
676    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
677        f.debug_struct("TaskRunner")
678            .field("repo", &self.repo)
679            .field("orch_home", &self.orch_home)
680            .finish_non_exhaustive()
681    }
682}
683
684impl TaskRunner {
685    pub fn new(repo: String) -> Self {
686        let orch_home = home::orch_home().unwrap_or_else(|e| {
687            tracing::warn!(error = %e, "orch_home() failed — falling back to /tmp/.orch");
688            PathBuf::from("/tmp").join(".orch")
689        });
690
691        Self {
692            repo,
693            orch_home,
694            store: None,
695        }
696    }
697
698    /// Set the unified task store for run audit trail.
699    #[must_use]
700    pub fn with_store(mut self, store: Arc<TaskStore>) -> Self {
701        self.store = Some(store);
702        self
703    }
704
705    /// Read a field from the task store.
706    async fn get_field(&self, task_id: &str, field: &str) -> Option<String> {
707        let task = store::opt_store_get_task(&self.store, &self.repo, task_id).await?;
708        match field {
709            "route_reason" => Some(task.route_reason),
710            "worktree" => Some(task.worktree),
711            "attempts" => Some(task.attempts.to_string()),
712            "delegations" => serde_json::to_string(&task.delegations).ok(),
713            "summary" => Some(task.summary),
714            "last_error" => Some(task.last_error),
715            "agent" => task.agent,
716            _ => None,
717        }
718    }
719
720    /// Keep a stored fallback on reroute, wipe agent/model when the run's agent is still stored
721    pub(crate) async fn clear_agent_on_reroute_unless_fallback(
722        &self,
723        task_id: &str,
724        ran_agent: &str,
725    ) {
726        let stored = store::opt_store_get_task(&self.store, &self.repo, task_id).await;
727        if Self::reroute_keeps_stored_agent(stored.as_ref(), ran_agent) {
728            tracing::debug!(
729                task_id,
730                "stored fallback agent present, skipping agent/model clear"
731            );
732            return;
733        }
734        if let Err(e) = store::store_set_result(
735            &self.store,
736            &self.repo,
737            task_id,
738            &[
739                ("agent", serde_json::json!("")),
740                ("model", serde_json::json!("")),
741            ],
742        )
743        .await
744        {
745            tracing::warn!(task_id, err = %e, "failed to clear agent/model for reroute in store");
746        }
747    }
748
749    /// True when a fallback is already stored and must survive this reroute
750    fn reroute_keeps_stored_agent(stored: Option<&store::Task>, ran_agent: &str) -> bool {
751        let Some(task) = stored else {
752            return false;
753        };
754        if !matches!(task.status, TaskStatus::InProgress) {
755            return true;
756        }
757        let stored_agent = task.agent.as_deref().unwrap_or("");
758        !stored_agent.is_empty() && stored_agent != ran_agent
759    }
760
761    async fn build_run_audit(&self, input: RunAuditInput<'_>) -> RunAudit {
762        // Filter empty strings so that a blank last_error does not shadow the
763        // actual parse_result error.  Without this, `Option::or(Some(""))` would
764        // return `Some("")` and prevent the `unwrap_or_else` fallback from ever
765        // running, causing task_runs.error to be written as "" even when the
766        // agent returned a classifiable error (fixes issue #2527).
767        let last_error = self
768            .get_field(input.task_id, "last_error")
769            .await
770            .filter(|s| !s.is_empty());
771        let outcome =
772            classify_run_outcome(input.status, input.parse_result, input.push_failed).to_string();
773
774        let error = if outcome == "success" {
775            String::new()
776        } else {
777            // Prefer this run's parsed error over a last_error left by a previous
778            // agent's failover. last_error stays the Err/EarlyReturn fallback (#3851)
779            let error = input
780                .error_override
781                .as_deref()
782                .filter(|s| !s.is_empty())
783                .map(str::to_string)
784                .or_else(|| match input.parse_result {
785                    Ok(parsed) => parsed
786                        .response
787                        .error
788                        .as_deref()
789                        .filter(|s| !s.is_empty())
790                        .map(str::to_string),
791                    Err(_) => None,
792                })
793                .or(last_error)
794                .unwrap_or_else(|| match input.parse_result {
795                    Ok(_) => String::new(),
796                    Err(err) => err.to_string(),
797                });
798
799            if error.is_empty() {
800                if input.status == "blocked" {
801                    match input.parse_result {
802                        Ok(parsed) if !parsed.response.summary.is_empty() => {
803                            parsed.response.summary.clone()
804                        }
805                        _ => "task blocked without explicit error".to_string(),
806                    }
807                } else if matches!(
808                    input.status,
809                    "done"
810                        | "completed"
811                        | "in_progress"
812                        | "in_review"
813                        | "needs_review"
814                        | "routed"
815                        | "new"
816                ) {
817                    // Canonical non-success statuses have no error message (e.g. "routed").
818                    String::new()
819                } else {
820                    // Non-canonical status with no error message — surface the status
821                    // explicitly so operators can identify which statuses need to be
822                    // added to the normalization map in parser.rs.
823                    format!("unrecognized status: {}", input.status)
824                }
825            } else {
826                error
827            }
828        };
829
830        let total_cost_usd = input
831            .parse_result
832            .as_ref()
833            .ok()
834            .and_then(|p| p.cost_usd)
835            .unwrap_or(0.0);
836
837        let (input_tokens, output_tokens) = extract_run_tokens(input.parse_result);
838
839        #[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
840        let duration_secs = input
841            .elapsed_secs
842            .map_or_else(|| run_duration_secs(input.started_at), |s| s as f64);
843
844        RunAudit {
845            stdout: input.raw_stdout.to_string(),
846            stderr: input.raw_stderr.to_string(),
847            parsed_response: serialize_parsed_response(input.parse_result, input.raw_stdout),
848            outcome,
849            error,
850            input_tokens,
851            output_tokens,
852            total_cost_usd,
853            duration_secs,
854        }
855    }
856
857    /// The blocked outcome for a sandbox that cannot start. No agent ran, so none is cooled down
858    async fn sandbox_blocked(&self, task_id: &str, reason: String) -> RunExecution {
859        tracing::error!(task_id, error = %reason, "blocking task");
860        if let Err(e) = store::store_set_result(
861            &self.store,
862            &self.repo,
863            task_id,
864            &[
865                ("last_error", serde_json::json!(reason)),
866                ("block_reason", serde_json::json!(reason)),
867            ],
868        )
869        .await
870        {
871            tracing::warn!(task_id, err = %e, "failed to write the sandbox block_reason");
872        }
873        RunExecution {
874            status: "blocked".to_string(),
875            exit_code: None,
876            audit: RunAudit {
877                stdout: String::new(),
878                stderr: String::new(),
879                parsed_response: String::new(),
880                outcome: "failed".to_string(),
881                error: reason,
882                input_tokens: 0,
883                output_tokens: 0,
884                total_cost_usd: 0.0,
885                duration_secs: 0.0,
886            },
887        }
888    }
889
890    /// Run a task through the full execution pipeline.
891    ///
892    /// Returns `Ok(None)` if the runner guard skipped the task (caller should not
893    /// post stale results). Returns `Ok(Some(status))` with the final task status.
894    pub async fn run(
895        &self,
896        task_id: &str,
897        agent: Option<&str>,
898        model: Option<&str>,
899        backend: Option<&dyn ExternalBackend>,
900        started_at: &chrono::DateTime<Utc>,
901    ) -> anyhow::Result<Option<RunExecution>> {
902        tracing::info!(
903            task_id,
904            agent = agent.unwrap_or("default"),
905            model = model.unwrap_or("default"),
906            "starting task execution"
907        );
908
909        // Check task guards; returns outcome indicating whether to proceed.
910        let attempts = match task_init::check_guards(task_id, &self.repo, &self.store).await {
911            Ok(task_init::GuardOutcome::Proceed(a)) => a,
912            Ok(task_init::GuardOutcome::Skip) => {
913                return Ok(None);
914            }
915            Ok(task_init::GuardOutcome::MaxAttempts) => {
916                // The caller marks the task blocked from the returned status, a task that ran out
917                // of attempts did not finish, so it must never reach done
918                if backend.is_some() {
919                    // If this was label-forced to a specific agent, remove the agent label
920                    // so that /retry can route to a different agent instead of looping.
921                    let route_reason = self
922                        .get_field(task_id, "route_reason")
923                        .await
924                        .unwrap_or_default();
925                    if route_reason.starts_with("label agent:") {
926                        let agent_label = route_reason.trim_start_matches("label ");
927                        let gh = GhHttp::new()?;
928                        match gh.remove_label(&self.repo, task_id, agent_label).await {
929                            Err(e) => {
930                                tracing::warn!(task_id, label = agent_label, err = %e, "failed to remove agent label after max attempts");
931                            }
932                            _ => {
933                                tracing::info!(
934                                    task_id,
935                                    label = agent_label,
936                                    "removed forced agent label after max attempts — /retry will use free routing"
937                                );
938                            }
939                        }
940                    }
941                }
942                return Ok(Some(RunExecution {
943                    status: "blocked".to_string(),
944                    exit_code: None,
945                    audit: RunAudit {
946                        stdout: String::new(),
947                        stderr: String::new(),
948                        parsed_response: String::new(),
949                        outcome: "failed".to_string(),
950                        error: "max attempts reached".to_string(),
951                        input_tokens: 0,
952                        output_tokens: 0,
953                        total_cost_usd: 0.0,
954                        duration_secs: 0.0,
955                    },
956                }));
957            }
958            Err(e) => return Err(e),
959        };
960
961        // Resolve project directory
962        let project_dir = self.resolve_project_dir().await?;
963
964        // Prepare task: worktree, context, prompts, and agent invocation
965        let init = task_init::prepare_task(
966            task_id,
967            agent,
968            model,
969            backend,
970            &self.repo,
971            &project_dir,
972            attempts,
973            &self.store,
974        )
975        .await?;
976
977        // A sandbox that cannot start blocks the task, which never runs without it. It is the
978        // operator's setup that fails, so no agent or model is cooled down
979        if init.invocation.sandbox
980            && let Err(e) = sandbox::argv(
981                &init.invocation,
982                &env::temp_dir().join("orch-sandbox-preflight"),
983            )
984            .await
985        {
986            return Ok(Some(self.sandbox_blocked(task_id, e.to_string()).await));
987        }
988
989        // Run agent session: spawn in tmux, wait for completion, collect output
990        let (tmux, tmux_session, session_output) = session::run_agent_session(
991            task_id,
992            &init.invocation,
993            &init.attempt_dir,
994            &self.orch_home,
995        )
996        .await;
997
998        // The pre-flight policy passed but the one built at spawn did not: block the same way
999        if let Some(reason) = session_output.sandbox_error.clone() {
1000            return Ok(Some(self.sandbox_blocked(task_id, reason).await));
1001        }
1002
1003        // Touch updated_at immediately after session exit so the stuck-task recovery
1004        // timer resets before response_handler post-processing (git push, PR creation,
1005        // status update) begins.  Without this, a session that ran longer than
1006        // no_session_stuck_timeout (default 10 min) would be re-dispatched by the tick
1007        // while the runner is still writing results.
1008        store::store_touch_updated_at(&self.store, &self.repo, task_id).await;
1009
1010        // If silence detection (tick phase1b) already rerouted this task while the
1011        // session was running, skip all fallback/review processing. Without this check the
1012        // runner would call `fallback::handle_error` on the empty output, exhaust fallback
1013        // agents, and mark the task `needs_review` — overwriting the rerouted status and
1014        // triggering a spurious review cycle.
1015        // Silence detection now sets Routed (with fallback agent) or NeedsReview (no fallback).
1016        // Check for both, plus New for backward compatibility.
1017        if let Some(stored) = store::opt_store_get_task(&self.store, &self.repo, task_id).await {
1018            let silence_reset = matches!(
1019                stored.status,
1020                TaskStatus::New | TaskStatus::Routed | TaskStatus::NeedsReview
1021            );
1022            if silence_reset {
1023                let status_str = stored.status.as_str();
1024                tracing::info!(
1025                    task_id,
1026                    status = status_str,
1027                    "task already reset by the engine — skipping fallback processing"
1028                );
1029                session::cleanup_session(task_id, &tmux, &tmux_session).await;
1030                let silence_err = agents::patterns::classify_from_text(0, "silence detected");
1031                let silence_parse: Result<agents::ParsedResponse, agents::AgentError> =
1032                    Err(silence_err);
1033                let audit = self
1034                    .build_run_audit(RunAuditInput {
1035                        task_id,
1036                        status: status_str,
1037                        parse_result: &silence_parse,
1038                        raw_stdout: &session_output.raw_stdout,
1039                        raw_stderr: &session_output.raw_stderr,
1040                        started_at,
1041                        error_override: Some(format!("task was reset by the engine while the session ran (status {status_str})")),
1042                        elapsed_secs: session_output.elapsed_secs,
1043                        push_failed: false,
1044                    })
1045                    .await;
1046                return Ok(Some(RunExecution {
1047                    status: status_str.to_string(),
1048                    exit_code: Some(session_output.exit_code),
1049                    audit,
1050                }));
1051            }
1052        }
1053
1054        // Log raw output for debugging agent failures
1055        let stdout_len = session_output.raw_stdout.len();
1056        let stderr_len = session_output.raw_stderr.len();
1057        let stdout_tail: String = session_output
1058            .raw_stdout
1059            .chars()
1060            .rev()
1061            .take(500)
1062            .collect::<String>()
1063            .chars()
1064            .rev()
1065            .collect();
1066        let stderr_tail: String = session_output
1067            .raw_stderr
1068            .chars()
1069            .rev()
1070            .take(500)
1071            .collect::<String>()
1072            .chars()
1073            .rev()
1074            .collect();
1075        tracing::info!(
1076            task_id,
1077            exit_code = session_output.exit_code,
1078            stdout_len,
1079            stderr_len,
1080            stdout_tail = %stdout_tail,
1081            stderr_tail = %stderr_tail,
1082            "agent raw output"
1083        );
1084
1085        // Use agent-specific parsing where possible. Previously we only attempted
1086        // parse_success_output when exit_code == 0 which caused runs that emitted
1087        // valid NDJSON but exited non-zero (kimi / minimax) to be classed as
1088        // failures. Try parsing success output first when stdout is present; if
1089        // parsing succeeds treat as success regardless of exit code. Only fall
1090        // back to error classification when parsing fails and the process exit
1091        // code indicates an error.
1092        for window in agents::claude::rate_limit_windows(&session_output.raw_stdout) {
1093            cooldown::record_rate_limit_window(&init.agent_name, &window).await;
1094        }
1095        let agent_runner = agents::get_runner(&init.agent_name);
1096        let parse_result =
1097            parse_session_output(task_id, &init.agent_name, &*agent_runner, &session_output);
1098
1099        // Write structured result.json for deterministic testing and debugging
1100        response_handler::write_result_json(
1101            &init.attempt_dir,
1102            task_id,
1103            &init.agent_name,
1104            init.model_name.as_deref(),
1105            session_output.exit_code,
1106            init.new_attempts,
1107            &parse_result,
1108            &session_output.raw_stdout,
1109            &session_output.raw_stderr,
1110        )
1111        .await;
1112
1113        // Handle outcome: success or error recovery
1114        // `fallback_error` carries the attributed error message from `handle_error()` when
1115        // the Continue path is taken.  It is passed as `error_override` to `build_run_audit`
1116        // so that stale `last_error` values from a previous agent's run are never written
1117        // into the current run's audit record (fixes misattribution bug: task_run shows
1118        // agent=kimi but error="minimax timed out...").
1119        let (final_status, push_failed, fallback_error) = match parse_result {
1120            Ok(ref parsed) => {
1121                let (status, push_failed) = response_handler::handle_success(
1122                    task_id,
1123                    parsed.clone(),
1124                    &init.wt,
1125                    &init.task_title,
1126                    &init.agent_name,
1127                    init.model_name.as_deref(),
1128                    init.new_attempts,
1129                    &self.repo,
1130                    &self.store,
1131                    &session_output.raw_stdout,
1132                )
1133                .await?;
1134                (status, push_failed, None)
1135            }
1136            Err(ref agent_err) => {
1137                match fallback::handle_error(
1138                    task_id,
1139                    agent_err,
1140                    &init.agent_name,
1141                    &*agent_runner,
1142                    init.model_name.as_deref(),
1143                    init.complexity.as_deref(),
1144                    init.new_attempts,
1145                    &self.store,
1146                    &self.repo,
1147                )
1148                .await?
1149                {
1150                    fallback::ErrorHandleResult::EarlyReturn { status } => {
1151                        // Task rerouted — clean up tmux session and env secrets
1152                        session::cleanup_session(task_id, &tmux, &tmux_session).await;
1153                        // The fallback handler already writes a sanitized `last_error`
1154                        // into the store for EarlyReturn paths (reroutes / needs_review).
1155                        // Passing the raw agent_err here would record noisy provider
1156                        // payloads (e.g. full api_retry NDJSON). Leave error_override
1157                        // as None so build_run_audit will prefer the store's last_error
1158                        // (see src/engine/runner/fallback.rs which persists a compact
1159                        // summary for rate-limit and reroute cases).
1160                        let audit = self
1161                            .build_run_audit(RunAuditInput {
1162                                task_id,
1163                                status: &status,
1164                                parse_result: &parse_result,
1165                                raw_stdout: &session_output.raw_stdout,
1166                                raw_stderr: &session_output.raw_stderr,
1167                                started_at,
1168                                error_override: None,
1169                                elapsed_secs: session_output.elapsed_secs,
1170                                push_failed: false,
1171                            })
1172                            .await;
1173                        return Ok(Some(RunExecution {
1174                            status,
1175                            exit_code: Some(session_output.exit_code),
1176                            audit,
1177                        }));
1178                    }
1179                    fallback::ErrorHandleResult::Continue { status, error } => {
1180                        (status, false, Some(error))
1181                    }
1182                }
1183            }
1184        };
1185
1186        // Kill tmux session if still alive
1187        session::cleanup_session(task_id, &tmux, &tmux_session).await;
1188
1189        // For no-op done tasks (agent finished with no code changes and no PR),
1190        // clean up the worktree and branch immediately — tmux session is already
1191        // dead so the tmux guard won't block removal.
1192        if final_status.as_str() == "done"
1193            && let Some(ref st) = self.store
1194            && let Err(e) = cleanup::cleanup_task_worktree(task_id, &self.repo, st).await
1195        {
1196            tracing::warn!(task_id, error = ?e, "worktree cleanup failed after no-op done");
1197        }
1198
1199        // Return final status and the session exit code so callers can record it.
1200        // Use `fallback_error` as error_override when available: it carries the error message
1201        // attributed to the current agent (set by handle_error), preventing stale last_error
1202        // values from a prior run from contaminating this run's audit record.
1203        let audit = self
1204            .build_run_audit(RunAuditInput {
1205                task_id,
1206                status: &final_status,
1207                parse_result: &parse_result,
1208                raw_stdout: &session_output.raw_stdout,
1209                raw_stderr: &session_output.raw_stderr,
1210                started_at,
1211                error_override: fallback_error,
1212                elapsed_secs: session_output.elapsed_secs,
1213                push_failed,
1214            })
1215            .await;
1216
1217        Ok(Some(RunExecution {
1218            status: final_status.clone(),
1219            exit_code: Some(session_output.exit_code),
1220            audit,
1221        }))
1222    }
1223
1224    /// Record task execution metrics to the database.
1225    #[expect(
1226        clippy::too_many_arguments,
1227        reason = "a metric row needs the agent, model, route, timing and outcome together"
1228    )]
1229    async fn record_metrics(
1230        &self,
1231        task_id: &str,
1232        agent_name: &str,
1233        model_name: Option<&str>,
1234        route_result: Option<&RouteResult>,
1235        started_at: &chrono::DateTime<Utc>,
1236        attempts: u32,
1237        final_status: &str,
1238        error_type: Option<&str>,
1239    ) {
1240        let completed_at = Utc::now();
1241        #[expect(clippy::cast_precision_loss, reason = "a run duration in seconds")]
1242        let duration_seconds = (completed_at - *started_at).num_milliseconds() as f64 / 1000.0;
1243
1244        // Classify outcome based on the final status and optional error_type/last_error
1245        let outcome = match final_status {
1246            "done" | "in_progress" | "in_review" => "success",
1247            "new" => "rerouted",
1248            "needs_review" => classify_run_error_type(error_type.unwrap_or("")),
1249            _ => "unknown",
1250        };
1251
1252        let complexity = route_result.map(|r| r.complexity.clone());
1253        let files_changed = git_ops::count_changed_files(&PathBuf::from(
1254            self.get_field(task_id, "worktree")
1255                .await
1256                .unwrap_or_default(),
1257        ))
1258        .await
1259        .unwrap_or(0);
1260
1261        if let Some(ref store) = self.store {
1262            // Only set error_type for non-success outcomes
1263            let db_error_type: Option<String> = if outcome == "success" {
1264                None
1265            } else {
1266                Some(outcome.to_string())
1267            };
1268
1269            let usage = store::get_token_usage(&self.store, &self.repo, task_id).await;
1270            let total_cost = store::get_reported_cost(&self.store, &self.repo, task_id).await;
1271            let input_tokens = if usage.input_tokens > 0 {
1272                Some(i64::try_from(usage.input_tokens).unwrap_or(i64::MAX))
1273            } else {
1274                None
1275            };
1276            let output_tokens = if usage.output_tokens > 0 {
1277                Some(i64::try_from(usage.output_tokens).unwrap_or(i64::MAX))
1278            } else {
1279                None
1280            };
1281
1282            let metric = InsertTaskMetric {
1283                repo: &self.repo,
1284                task_id,
1285                agent: agent_name,
1286                model: model_name,
1287                complexity: complexity.as_deref(),
1288                outcome,
1289                duration_seconds,
1290                started_at,
1291                completed_at: &completed_at,
1292                attempts: i32::try_from(attempts.saturating_add(1)).unwrap_or(i32::MAX),
1293                files_changed: i32::try_from(files_changed).unwrap_or(i32::MAX),
1294                error_type: db_error_type.as_deref(),
1295                input_tokens,
1296                output_tokens,
1297                total_cost_usd: total_cost,
1298            };
1299
1300            if let Err(e) = store.insert_task_metric(&metric).await {
1301                tracing::error!(task_id, ?e, "failed to record task metrics");
1302            }
1303        }
1304    }
1305
1306    /// Run a task with full engine context (backend, tmux, capture).
1307    ///
1308    /// Called by the engine dispatch loop with richer context.
1309    /// Returns a `WeightSignal` for the engine to feed back to the router.
1310    pub async fn run_with_context(
1311        &self,
1312        task: &ExternalTask,
1313        backend: &Arc<dyn ExternalBackend>,
1314        _tmux: &Arc<TmuxManager>,
1315        route_result: Option<&RouteResult>,
1316    ) -> anyhow::Result<WeightSignal> {
1317        let task_id = &task.id.0;
1318        let agent = route_result.map(|r| r.agent.as_str());
1319        let agent_name = agent.unwrap_or("claude").to_string();
1320        let model = route_result.and_then(|r| r.model.as_deref());
1321
1322        // Record start time for metrics (before any work begins)
1323        let started_at = Utc::now();
1324
1325        // Store task info for prompt building (title/body already in store tasks table)
1326
1327        // Record run start in task_runs audit trail
1328        let run_audit_id = if let Some(ref store) = self.store {
1329            let attempt: i32 = self
1330                .get_field(task_id, "attempts")
1331                .await
1332                .and_then(|s| s.parse().ok())
1333                .unwrap_or(0)
1334                + 1;
1335            match store.resolve_task_id(&self.repo, task_id).await {
1336                Ok(Some(store_id)) => {
1337                    match store
1338                        .start_run(&crate::store::StartRun {
1339                            task_id: store_id,
1340                            attempt,
1341                            run_type: "agent",
1342                            agent: &agent_name,
1343                            model: model.unwrap_or(""),
1344                            command: &format!(
1345                                "{} --model {}",
1346                                agent_name,
1347                                model.unwrap_or("default")
1348                            ),
1349                            prompt: &task.body,
1350                        })
1351                        .await
1352                    {
1353                        Ok(run_id) => Some(run_id),
1354                        Err(e) => {
1355                            tracing::warn!(
1356                                task_id,
1357                                error = %e,
1358                                "failed to record run start in audit trail"
1359                            );
1360                            None
1361                        }
1362                    }
1363                }
1364                _ => None,
1365            }
1366        } else {
1367            None
1368        };
1369
1370        // Run the task.
1371        // Always finalize the run audit row, even on early-return/error paths.
1372        let run_result = match self
1373            .run(task_id, agent, model, Some(&**backend), &started_at)
1374            .await
1375        {
1376            Ok(run_result) => run_result,
1377            Err(e) => {
1378                if let Some(run_id) = run_audit_id
1379                    && let Some(ref store) = self.store
1380                    && let Err(complete_err) = store
1381                        .complete_run(&crate::store::CompleteRun {
1382                            run_id,
1383                            exit_code: Some(-1),
1384                            stdout: "",
1385                            stderr: "",
1386                            parsed: "",
1387                            outcome: "failed",
1388                            error: &e.to_string(),
1389                            tokens: RunTokenUsage::default(),
1390                        })
1391                        .await
1392                {
1393                    tracing::warn!(
1394                        task_id,
1395                        run_id,
1396                        error = %complete_err,
1397                        "failed to record run completion in audit trail after runner error"
1398                    );
1399                }
1400                return Err(e);
1401            }
1402        };
1403
1404        // If the runner guard skipped the task, do not re-post stale data as a new comment.
1405        let (status, exit_code_opt, run_audit) = if let Some(result) = run_result {
1406            (result.status, result.exit_code, result.audit)
1407        } else {
1408            if let Some(run_id) = run_audit_id
1409                && let Some(ref store) = self.store
1410                && let Err(e) = store
1411                    .complete_run(&crate::store::CompleteRun {
1412                        run_id,
1413                        exit_code: Some(-1),
1414                        stdout: "",
1415                        stderr: "",
1416                        parsed: "",
1417                        outcome: "aborted",
1418                        error: "run skipped by guard",
1419                        tokens: RunTokenUsage::default(),
1420                    })
1421                    .await
1422            {
1423                tracing::warn!(
1424                    task_id,
1425                    run_id,
1426                    error = %e,
1427                    "failed to record skipped run completion in audit trail"
1428                );
1429            }
1430            tracing::info!(task_id, "guard skipped task — not posting stale result");
1431            return Ok(WeightSignal::None);
1432        };
1433
1434        // Process delegations if the agent requested subtasks
1435        let delegations_raw = self
1436            .get_field(task_id, "delegations")
1437            .await
1438            .unwrap_or_default();
1439        if !delegations_raw.is_empty() {
1440            match serde_json::from_str::<Vec<Delegation>>(&delegations_raw) {
1441                Ok(delegations) if !delegations.is_empty() => {
1442                    self.process_delegations(task, &delegations, backend)
1443                        .await?;
1444                    if let Err(e) = store::store_set_result(
1445                        &self.store,
1446                        &self.repo,
1447                        task_id,
1448                        &[("delegations", serde_json::json!([]))],
1449                    )
1450                    .await
1451                    {
1452                        tracing::warn!(task_id, err = %e, "failed to clear delegations in store");
1453                    }
1454                }
1455                Ok(_) => {} // empty list, nothing to do
1456                Err(e) => {
1457                    tracing::error!(task_id, error = %e, "corrupt delegations JSON — clearing");
1458                    if let Err(e2) = store::store_set_result(
1459                        &self.store,
1460                        &self.repo,
1461                        task_id,
1462                        &[("delegations", serde_json::json!([]))],
1463                    )
1464                    .await
1465                    {
1466                        tracing::warn!(task_id, err = %e2, "failed to clear corrupt delegations in store");
1467                    }
1468                    if let Err(e2) = store::store_set_result(
1469                        &self.store,
1470                        &self.repo,
1471                        task_id,
1472                        &[(
1473                            "last_error",
1474                            serde_json::json!(format!("delegation parse failed: {e}")),
1475                        )],
1476                    )
1477                    .await
1478                    {
1479                        tracing::warn!(task_id, err = %e2, "failed to write delegation_parse_failed last_error to store");
1480                    }
1481                }
1482            }
1483        }
1484
1485        // Post result to GitHub
1486        let summary = self.get_field(task_id, "summary").await.unwrap_or_default();
1487        let last_error = self
1488            .get_field(task_id, "last_error")
1489            .await
1490            .unwrap_or_default();
1491
1492        // Determine weight signal based on outcome.
1493        // Any reroute ("new"/"routed") should avoid review-gate and go back through
1494        // normal dispatch, regardless of why it was rerouted.
1495        let is_rerouted = status == "new" || status == "routed";
1496        if is_rerouted {
1497            self.clear_agent_on_reroute_unless_fallback(task_id, &agent_name)
1498                .await;
1499        }
1500        // Determine weight signal based on outcome.
1501        // Reroutes should only emit RateLimited when the underlying error is genuinely
1502        // a rate limit (not silence detection, timeouts, parse errors, etc.). This prevents
1503        // false cooldown cascades where non-rate-limit failures are treated as rate limits.
1504        let is_rate_limit_error = classify_run_error_type(&last_error) == "rate_limit";
1505        let weight_signal = if is_rerouted {
1506            if is_rate_limit_error {
1507                WeightSignal::RateLimited {
1508                    agent: agent_name.clone(),
1509                }
1510            } else {
1511                // Non-rate-limit reroutes (silence detection, timeout, parse error): signal
1512                // the engine to keep the task in "new" for re-routing. Using Rerouted (not
1513                // None) prevents tick from converting the silence-reset back to needs_review,
1514                // which would trigger the no-code review shortcut and falsely mark it done.
1515                WeightSignal::Rerouted
1516            }
1517        } else if status == "needs_review" {
1518            // Rate limit errors that escalate to needs_review must NOT reset backoff
1519            // counters (issue #2153).  These errors indicate the agent is genuinely
1520            // unavailable — resetting would wipe the cooldown and cause the next dispatch
1521            // to retry the same rate-limited agent immediately.
1522            //
1523            // Detection: classify_run_error_type() confirms "rate limit" in last_error.
1524            // Reuse the same classifier as `is_rate_limit_error` above rather than a raw
1525            // substring check — billing-cycle exhaustion messages (e.g. Kimi's "usage
1526            // limit for this billing cycle") also contain "usage limit"/"rate limit", so
1527            // a bare substring check would record a rate-limit event into the router's
1528            // health signal for a condition that isn't a genuine rate limit (issue #3529).
1529            let signal = needs_review_signal(&last_error, &agent_name);
1530            match &signal {
1531                WeightSignal::RateLimited { .. } => {
1532                    // Record in the metrics store so the router's health check detects it.
1533                    if let Some(ref store) = self.store
1534                        && let Err(e) = store
1535                            .record_rate_limit(&agent_name, "rate", Some(task_id))
1536                            .await
1537                    {
1538                        tracing::warn!(task_id, agent = %agent_name, err = %e, "failed to persist rate_limit event — router health check may miss this signal");
1539                    }
1540                    // Also record via the cooldown system so model_for_complexity() skips
1541                    // this model.
1542                    cooldown::record_rate_limit(&agent_name, model, &last_error).await;
1543                }
1544                WeightSignal::Success { .. } => {
1545                    // Reset backoff so the next failure starts fresh.
1546                    cooldown::record_agent_success(&agent_name, model.unwrap_or("")).await;
1547                }
1548                _ => {}
1549            }
1550            signal
1551        } else if status == "done" || status == "in_progress" || status == "in_review" {
1552            // Reset exponential backoff counters so the next failure starts fresh.
1553            cooldown::record_agent_success(&agent_name, model.unwrap_or("")).await;
1554            WeightSignal::Success {
1555                agent: agent_name.clone(),
1556            }
1557        } else if status == "blocked" {
1558            WeightSignal::Blocked
1559        } else {
1560            WeightSignal::None
1561        };
1562
1563        // Record metrics
1564        {
1565            let attempts: u32 = self
1566                .get_field(task_id, "attempts")
1567                .await
1568                .and_then(|s| s.parse().ok())
1569                .unwrap_or(0);
1570            self.record_metrics(
1571                task_id,
1572                &agent_name,
1573                model,
1574                route_result,
1575                &started_at,
1576                attempts,
1577                &status,
1578                if last_error.is_empty() {
1579                    None
1580                } else {
1581                    Some(&last_error)
1582                },
1583            )
1584            .await;
1585        }
1586
1587        // Complete run in task_runs audit trail (include exit code from runner when available)
1588        if let Some(run_id) = run_audit_id
1589            && let Some(ref store) = self.store
1590            && let Err(e) = store
1591                .complete_run(&crate::store::CompleteRun {
1592                    run_id,
1593                    exit_code: exit_code_opt,
1594                    stdout: &run_audit.stdout,
1595                    stderr: &run_audit.stderr,
1596                    parsed: &run_audit.parsed_response,
1597                    outcome: &run_audit.outcome,
1598                    error: &run_audit.error,
1599                    tokens: RunTokenUsage {
1600                        input_tokens: run_audit.input_tokens,
1601                        output_tokens: run_audit.output_tokens,
1602                        total_cost_usd: run_audit.total_cost_usd,
1603                        duration_secs: run_audit.duration_secs,
1604                    },
1605                })
1606                .await
1607        {
1608            tracing::warn!(
1609                task_id,
1610                run_id,
1611                error = %e,
1612                "failed to record run completion in audit trail"
1613            );
1614        }
1615
1616        // If task was rerouted (status=routed or new after run), update GitHub agent label
1617        // so the router doesn't re-route back to the same failed agent.
1618        if is_rerouted && !is_internal_id(task_id) {
1619            let new_agent = self.get_field(task_id, "agent").await.unwrap_or_default();
1620            if !new_agent.is_empty() && new_agent != agent_name {
1621                update_agent_label_after_reroute(
1622                    &self.repo,
1623                    task,
1624                    backend,
1625                    &agent_name,
1626                    &new_agent,
1627                )
1628                .await;
1629            }
1630        }
1631
1632        // Internal tasks: skip GitHub status/comment updates — dispatch phase handles status.
1633        if is_internal_id(task_id) {
1634            return Ok(weight_signal);
1635        }
1636
1637        // Update GitHub status
1638        let new_status = match status.as_str() {
1639            "done" => Status::Done,
1640            "in_progress" => Status::InProgress,
1641            "in_review" => Status::InReview,
1642            "blocked" => Status::Blocked,
1643            "routed" => Status::Routed, // Rerouted (agent/model already set)
1644            "new" => Status::New,       // Legacy reroute path
1645            _ => Status::NeedsReview,
1646        };
1647        // Store-first: update SQLite (must succeed), then mirror to backend.
1648        if let Some(ref store) = self.store
1649            && let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &task.id.0).await
1650        {
1651            store
1652                .update_status(store_id, tasks::status_to_task_status(new_status))
1653                .await
1654                .context("store-first status update failed in runner")?;
1655        }
1656        if let Err(e) = backend.update_status(&task.id, new_status).await {
1657            tracing::warn!(
1658                task_id,
1659                ?new_status,
1660                err = %e,
1661                "failed to mirror runner status to backend — store is authoritative"
1662            );
1663        }
1664
1665        let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ");
1666        let mut comment = if !summary.is_empty() {
1667            format!("[{now}] {status}: {summary}")
1668        } else if !last_error.is_empty() {
1669            format!("[{now}] {status}: {last_error}")
1670        } else {
1671            format!("[{now}] {status}")
1672        };
1673
1674        // Append visible attribution footer (matches PR body style)
1675        let footer = engine::attribution_footer("Created", &agent_name, model);
1676        comment.push_str(&footer);
1677
1678        backend.post_comment(&task.id, &comment).await?;
1679
1680        Ok(weight_signal)
1681    }
1682
1683    /// Process delegations from an agent response.
1684    ///
1685    /// Creates child GitHub issues for each delegation and marks the parent
1686    /// as blocked. The engine's Phase 4 unblock mechanism will re-activate
1687    /// the parent when all children are done.
1688    async fn process_delegations(
1689        &self,
1690        parent_task: &ExternalTask,
1691        delegations: &[Delegation],
1692        backend: &Arc<dyn ExternalBackend>,
1693    ) -> anyhow::Result<()> {
1694        let parent_id = &parent_task.id;
1695
1696        // Fetch existing open tasks once for dedup — prevents creating duplicate
1697        // GitHub issues when the same delegations are produced on repeated runs.
1698        let existing_titles: HashSet<String> = match backend.list_all_tasks().await {
1699            Ok(tasks) => tasks.into_iter().map(|t| t.title).collect(),
1700            Err(e) => {
1701                tracing::warn!(err = %e, "failed to fetch tasks for dedup — will create all");
1702                HashSet::new()
1703            }
1704        };
1705
1706        for delegation in delegations {
1707            // Skip if an issue with the same title already exists
1708            if existing_titles.contains(&delegation.title) {
1709                tracing::info!(
1710                    parent = parent_id.0,
1711                    title = delegation.title,
1712                    "skipping delegation — issue with same title already exists"
1713                );
1714                continue;
1715            }
1716
1717            // Build labels: status:new + any labels from the delegation
1718            let mut labels = delegation.labels.clone();
1719            labels.push("status:new".to_string());
1720
1721            // Build child body with delegation reference
1722            let child_body = format!(
1723                "{}\n\n---\n_Delegated from #{}_{}",
1724                delegation.body,
1725                parent_id.0,
1726                engine::orch_footer()
1727            );
1728
1729            match backend
1730                .create_sub_task(parent_id, &delegation.title, &child_body, &labels)
1731                .await
1732            {
1733                Ok(child_id) => {
1734                    tracing::info!(
1735                        parent = parent_id.0,
1736                        child = child_id.0,
1737                        title = delegation.title,
1738                        "created delegated subtask"
1739                    );
1740                }
1741                Err(e) => {
1742                    tracing::error!(
1743                        parent = parent_id.0,
1744                        title = delegation.title,
1745                        err = %e,
1746                        "failed to create delegated subtask"
1747                    );
1748                }
1749            }
1750        }
1751
1752        // Mark parent as blocked — store-first, then mirror to backend.
1753        if let Some(ref store) = self.store
1754            && let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &parent_id.0).await
1755        {
1756            store
1757                .update_status(store_id, TaskStatus::Blocked)
1758                .await
1759                .context("store-first: failed to block parent in store")?;
1760        }
1761        if let Err(e) = backend.update_status(parent_id, Status::Blocked).await {
1762            tracing::warn!(
1763                parent = parent_id.0,
1764                err = %e,
1765                "failed to mirror blocked status to backend — store is authoritative"
1766            );
1767        }
1768
1769        // Post summary comment on parent
1770        let summary = delegations
1771            .iter()
1772            .enumerate()
1773            .map(|(i, d)| format!("{}. {}", i + 1, d.title))
1774            .collect::<Vec<_>>()
1775            .join("\n");
1776
1777        backend
1778            .post_comment(
1779                parent_id,
1780                &format!(
1781                    "Delegated {} subtask(s):\n\n{}\n\nParent task is blocked until all subtasks complete.{}",
1782                    delegations.len(),
1783                    summary,
1784                    engine::orch_footer()
1785                ),
1786            )
1787            .await?;
1788
1789        Ok(())
1790    }
1791
1792    /// Resolve the project directory for this repo.
1793    pub(crate) async fn resolve_project_dir(&self) -> anyhow::Result<PathBuf> {
1794        // Explicit env var always wins
1795        if let Ok(dir) = env::var("PROJECT_DIR")
1796            && !dir.is_empty()
1797        {
1798            return Ok(PathBuf::from(dir));
1799        }
1800
1801        // Check projects list in global config — find path for this repo
1802        if let Ok(paths) = config::get_project_paths() {
1803            for path_str in &paths {
1804                let path = PathBuf::from(path_str);
1805                // Check if this project's .orch.yml has matching repo
1806                if let Ok(repo) = config::get_repo_for_project(&path) {
1807                    // Use tokio::fs::metadata for async-safe existence check
1808                    let path_for_check = path.clone();
1809                    if repo == self.repo && fs::metadata(&path_for_check).await.is_ok() {
1810                        return Ok(path);
1811                    }
1812                }
1813            }
1814        }
1815
1816        // Legacy: check config project_dir
1817        if let Ok(dir) = config::get("project_dir")
1818            && !dir.is_empty()
1819        {
1820            let path = PathBuf::from(&dir);
1821            let path_for_check = path.clone();
1822            if fs::metadata(&path_for_check).await.is_ok() {
1823                return Ok(path);
1824            }
1825        }
1826
1827        // Check for bare clone
1828        let parts: Vec<&str> = self.repo.split('/').collect();
1829        if parts.len() == 2 {
1830            let bare = self
1831                .orch_home
1832                .join("projects")
1833                .join(parts[0])
1834                .join(format!("{}.git", parts[1]));
1835            let bare_for_check = bare.clone();
1836            if task::spawn_blocking(move || bare_for_check.exists())
1837                .await
1838                .unwrap_or(false)
1839            {
1840                return Ok(bare);
1841            }
1842        }
1843
1844        // No resolvable project directory: not in active config, no legacy
1845        // project_dir, no bare clone. Falling back to the service's own CWD
1846        // here would silently point worktree creation at an unrelated
1847        // directory (see issue #3416) — surface a clear error instead.
1848        Err(NoProjectDir {
1849            repo: self.repo.clone(),
1850            bare_clone: self
1851                .orch_home
1852                .join("projects")
1853                .join(self.repo.split('/').next().unwrap_or_default())
1854                .join(format!(
1855                    "{}.git",
1856                    self.repo.split('/').nth(1).unwrap_or_default()
1857                )),
1858        }
1859        .into())
1860    }
1861}
1862
1863/// Update the GitHub agent label on an external task when it is rerouted to a new agent.
1864///
1865/// Removes the old `agent:<old_agent>` label, ensures the new `agent:<new_agent>` label exists
1866/// in the repo, then applies it. Returns `true` only when both `ensure_label` (repo label creation)
1867/// and `set_labels` (issue label assignment) succeed. `remove_label` failures are non-fatal.
1868async fn update_agent_label_after_reroute(
1869    #[cfg_attr(
1870        test,
1871        expect(unused_variables, reason = "only the label call outside tests uses it")
1872    )]
1873    repo: &str,
1874    task: &ExternalTask,
1875    backend: &Arc<dyn ExternalBackend>,
1876    old_agent: &str,
1877    new_agent: &str,
1878) -> bool {
1879    let old_label = format!("agent:{old_agent}");
1880    if let Err(e) = backend.remove_label(&task.id, &old_label).await {
1881        tracing::warn!(
1882            task_id = ?task.id,
1883            label = old_label,
1884            error = %e,
1885            "failed to remove old agent label during re-route"
1886        );
1887    }
1888    let new_label = format!("agent:{new_agent}");
1889
1890    // ensure_label creates the repo-level label if missing (best-effort).
1891    // Skipped under #[cfg(test)] to avoid real GitHub API calls.
1892    #[cfg(not(test))]
1893    {
1894        match GhHttp::new() {
1895            Ok(gh) => {
1896                if gh
1897                    .ensure_label(
1898                        repo,
1899                        &new_label,
1900                        http::status_label_color(&new_label),
1901                        &format!("Agent: {new_agent}"),
1902                    )
1903                    .await
1904                    .is_err()
1905                {
1906                    tracing::warn!(
1907                        task_id = ?task.id,
1908                        label = %new_label,
1909                        "ensure_label failed during agent label update after failover"
1910                    );
1911                }
1912            }
1913            Err(e) => {
1914                tracing::warn!(
1915                    task_id = ?task.id,
1916                    err = %e,
1917                    "GhHttp::new() failed — skipping ensure_label"
1918                );
1919            }
1920        }
1921    }
1922
1923    if let Err(e) = backend
1924        .set_labels(&task.id, slice::from_ref(&new_label))
1925        .await
1926    {
1927        tracing::warn!(
1928            task_id = ?task.id,
1929            label = %new_label,
1930            error = %e,
1931            "set_labels failed during agent label update after failover"
1932        );
1933        return false;
1934    }
1935    true
1936}
1937
1938/// Weight signal for a run that ended in `needs_review`. `prepare_task` clears `last_error` at run
1939/// start, so a non-empty value means `handle_error` ended this run
1940fn needs_review_signal(last_error: &str, agent: &str) -> WeightSignal {
1941    let agent = agent.to_string();
1942    if classify_run_error_type(last_error) == "rate_limit" {
1943        WeightSignal::RateLimited { agent }
1944    } else if !last_error.is_empty() {
1945        WeightSignal::Failed { agent }
1946    } else {
1947        WeightSignal::Success { agent }
1948    }
1949}
1950
1951#[cfg(test)]
1952mod tests {
1953    use super::*;
1954    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
1955    use crate::engine::runner::agents::patterns::safe_tail as safe_utf8_tail;
1956    use crate::parser::AgentResponse;
1957    use crate::store::{NewTask, TaskStatus, TaskStore};
1958    use async_trait::async_trait;
1959    use chrono::Utc;
1960    use std::sync::{Arc, Mutex};
1961    use tempfile::TempDir;
1962    use tokio::process::Command;
1963
1964    static ENV_LOCK: Mutex<()> = Mutex::new(());
1965
1966    fn ok_parse_result(status: &str) -> agents::ParsedResponse {
1967        agents::ParsedResponse {
1968            response: AgentResponse {
1969                status: status.to_string(),
1970                summary: String::new(),
1971                accomplished: vec![],
1972                remaining: vec![],
1973                files: vec![],
1974                error: None,
1975                input_tokens: None,
1976                output_tokens: None,
1977                learnings: vec![],
1978                delegations: vec![],
1979                screenshots: vec![],
1980                screenshot_note: None,
1981            },
1982            input_tokens: None,
1983            output_tokens: None,
1984            cost_usd: None,
1985            duration_ms: None,
1986        }
1987    }
1988
1989    // ── safe_utf8_tail ───────────────────────────────────────────────────────
1990
1991    #[test]
1992    fn safe_utf8_tail_short_string_returned_as_is() {
1993        assert_eq!(safe_utf8_tail("hello", 100), "hello");
1994        assert_eq!(safe_utf8_tail("", 10), "");
1995    }
1996
1997    #[test]
1998    fn safe_utf8_tail_truncates_ascii() {
1999        let s = "abcdefghij"; // 10 bytes
2000        assert_eq!(safe_utf8_tail(s, 5), "fghij");
2001        assert_eq!(safe_utf8_tail(s, 10), "abcdefghij");
2002    }
2003
2004    #[test]
2005    fn safe_utf8_tail_handles_multibyte_boundary() {
2006        // "日本語" is 3 chars × 3 bytes each = 9 bytes total
2007        let s = "日本語";
2008        assert_eq!(s.len(), 9);
2009        // Cutting at 8 bytes lands in the middle of "語" (3 bytes at 6..9).
2010        // safe_utf8_tail should walk forward to the next boundary at 9 (end) rather
2011        // than panicking on an invalid slice.
2012        let tail = safe_utf8_tail(s, 8);
2013        assert!(
2014            s.ends_with(tail),
2015            "tail must be a valid suffix of the original"
2016        );
2017        assert!(
2018            str::from_utf8(tail.as_bytes()).is_ok(),
2019            "tail must be valid UTF-8"
2020        );
2021    }
2022
2023    #[test]
2024    fn safe_utf8_tail_handles_mixed_ascii_and_multibyte() {
2025        let s = "hello日本語world"; // ascii + CJK + ascii
2026        for max_bytes in [1, 3, 5, 7, 11, 13, 20, 100] {
2027            let tail = safe_utf8_tail(s, max_bytes);
2028            assert!(
2029                str::from_utf8(tail.as_bytes()).is_ok(),
2030                "max_bytes={max_bytes}: tail must be valid UTF-8"
2031            );
2032            assert!(
2033                s.ends_with(tail),
2034                "max_bytes={max_bytes}: tail must be a suffix of original"
2035            );
2036        }
2037    }
2038
2039    #[test]
2040    fn safe_utf8_tail_exact_boundary() {
2041        let s = "abcdef"; // 6 bytes
2042        assert_eq!(safe_utf8_tail(s, 6), "abcdef"); // exactly fits
2043        assert_eq!(safe_utf8_tail(s, 0), ""); // empty tail
2044    }
2045
2046    #[test]
2047    fn parse_success_output_synthesizes_done_from_plain_text() {
2048        let runner = agents::get_runner("claude");
2049        let parsed = parse_success_output(
2050            "123",
2051            "claude",
2052            &*runner,
2053            "No open positions, no trade executions, no conditions change.",
2054        )
2055        .unwrap();
2056
2057        assert_eq!(parsed.response.status, "done");
2058        assert_eq!(
2059            parsed.response.summary,
2060            "No open positions, no trade executions, no conditions change."
2061        );
2062    }
2063
2064    #[test]
2065    fn parse_success_output_synthesizes_needs_review_from_error_text() {
2066        let runner = agents::get_runner("claude");
2067        let parsed =
2068            parse_success_output("123", "claude", &*runner, "Failed to update branch").unwrap();
2069
2070        assert_eq!(parsed.response.status, "needs_review");
2071        assert_eq!(
2072            parsed.response.error.as_deref(),
2073            Some("agent returned plain text instead of JSON")
2074        );
2075    }
2076
2077    #[test]
2078    fn parse_success_output_keeps_empty_text_invalid() {
2079        let runner = agents::get_runner("claude");
2080        let err = parse_success_output("123", "claude", &*runner, "").unwrap_err();
2081
2082        assert!(matches!(err, agents::AgentError::InvalidResponse { .. }));
2083    }
2084
2085    #[test]
2086    fn parse_success_output_preserves_tokens_from_ndjson() {
2087        // Simulates NDJSON output from --output-format stream-json where
2088        // the result event contains token usage. parse_response fails on
2089        // NDJSON (expects single JSON blob), but find_agent_result extracts
2090        // tokens from the NDJSON envelope so cost tracking is preserved.
2091        let ndjson = r#"{"type":"system","subtype":"init","session_id":"abc"}
2092{"type":"assistant","message":{"content":[{"type":"text","text":"Done."}],"usage":{"input_tokens":100,"output_tokens":50}}}
2093{"type":"result","subtype":"success","is_error":false,"duration_ms":5000,"result":"All changes committed successfully","usage":{"input_tokens":1234,"output_tokens":567}}"#;
2094        let runner = agents::get_runner("claude");
2095        let parsed = parse_success_output("456", "claude", &*runner, ndjson).unwrap();
2096
2097        assert_eq!(parsed.response.status, "done");
2098        assert_eq!(parsed.input_tokens, Some(1234));
2099        assert_eq!(parsed.output_tokens, Some(567));
2100        assert_eq!(parsed.duration_ms, Some(5000));
2101    }
2102
2103    #[test]
2104    fn parse_success_output_preserves_tokens_from_minimax_ndjson() {
2105        // MiniMax/Kimi use assistant event fallback when there's no result event.
2106        let ndjson = r#"{"type":"system","subtype":"init"}
2107{"type":"assistant","message":{"content":[{"type":"text","text":"No open positions."}],"usage":{"input_tokens":2000,"output_tokens":300}}}"#;
2108        let runner = agents::get_runner("minimax");
2109        let parsed = parse_success_output("789", "minimax", &*runner, ndjson).unwrap();
2110
2111        assert_eq!(parsed.response.status, "done");
2112        assert_eq!(parsed.input_tokens, Some(2000));
2113        assert_eq!(parsed.output_tokens, Some(300));
2114    }
2115
2116    #[test]
2117    fn classify_run_error_type_recognizes_specific_patterns() {
2118        let cases = [
2119            ("push failed: remote rejected", "push_failed"),
2120            ("No commits between main and branch", "no_commits"),
2121            ("create PR failed: 422", "pr_failed"),
2122            ("invalid response from agent", "parse_error"),
2123            ("parse error: missing json", "parse_error"),
2124            ("exceeded max attempts while rerouting", "max_attempts"),
2125            ("unit test failed", "ci_failure"),
2126            ("build failed: type mismatch", "ci_failure"),
2127            ("lint failed on warning", "ci_failure"),
2128            ("3 tests failed", "ci_failure"),
2129            ("test suite failed: 3 tests failed", "ci_failure"),
2130            (
2131                "claude interrupted: exited without a result envelope",
2132                "interrupted",
2133            ),
2134        ];
2135
2136        for (input, expected) in cases {
2137            assert_eq!(classify_run_error_type(input), expected, "input={input}");
2138        }
2139    }
2140
2141    #[test]
2142    fn classify_run_error_type_falls_back_for_unknown_errors() {
2143        assert_eq!(classify_run_error_type("something unexpected"), "failed");
2144    }
2145
2146    #[test]
2147    fn classify_run_error_type_connection_error_mentioning_pull_request_is_not_pr_failed() {
2148        // A generic GitHub API or connection error whose message happens to contain the
2149        // substring "pull request" must NOT be classified as pr_failed.
2150        assert_eq!(
2151            classify_run_error_type("failed to fetch pull request details from GitHub"),
2152            "failed"
2153        );
2154        assert_eq!(
2155            classify_run_error_type("connection reset while reading pull request data"),
2156            "failed"
2157        );
2158    }
2159
2160    #[test]
2161    fn classify_run_error_type_url_with_test_substring_is_not_ci_failure() {
2162        // A connection error whose URL path contains "/test-org/" must NOT be ci_failure.
2163        assert_eq!(
2164            classify_run_error_type(
2165                "failed to connect to api.github.com/repos/test-org/repo/pulls/123"
2166            ),
2167            "failed"
2168        );
2169        // Bare "test" word in an unrelated error must not trigger ci_failure.
2170        assert_eq!(
2171            classify_run_error_type("context deadline exceeded testing connection"),
2172            "failed"
2173        );
2174    }
2175
2176    #[test]
2177    fn classify_run_error_type_usage_without_limit_is_not_rate_limit() {
2178        // "usage" alone (e.g. token usage reporting) must NOT be rate_limit.
2179        assert_eq!(
2180            classify_run_error_type("token usage: 1245 tokens"),
2181            "failed"
2182        );
2183        assert_eq!(
2184            classify_run_error_type("incorrect usage of git command"),
2185            "failed"
2186        );
2187        // Only "usage limit" should trigger rate_limit.
2188        assert_eq!(
2189            classify_run_error_type("exceeded usage limit for this billing period"),
2190            "rate_limit"
2191        );
2192    }
2193
2194    #[test]
2195    fn classify_run_error_type_billing_cycle_is_not_auth_error() {
2196        // "billing cycle" quota-exhaustion messages must be classified as billing_cycle_exhausted,
2197        // not auth_error — aligning with cooldown.rs CreditExhaustionReason::BillingCycleExhausted.
2198        assert_eq!(
2199            classify_run_error_type(
2200                "You've reached your usage limit for this billing cycle. \
2201                 Your quota will be refreshed in the next cycle."
2202            ),
2203            "billing_cycle_exhausted"
2204        );
2205        assert_eq!(
2206            classify_run_error_type("quota exhausted for billing cycle"),
2207            "billing_cycle_exhausted"
2208        );
2209        // Plain "billing" without "cycle" that used to map to auth_error now falls through to "failed".
2210        assert_eq!(
2211            classify_run_error_type("billing account suspended"),
2212            "failed"
2213        );
2214    }
2215
2216    #[test]
2217    fn classify_run_error_type_monthly_quota_is_billing_cycle() {
2218        // Regression test for issue #3302: GitHub Copilot "You have exceeded your
2219        // monthly quota" must map to billing_cycle_exhausted, not "failed".
2220        assert_eq!(
2221            classify_run_error_type(
2222                "opencode failed: agent failed: You have exceeded your monthly quota"
2223            ),
2224            "billing_cycle_exhausted"
2225        );
2226        assert_eq!(
2227            classify_run_error_type("You have exceeded your monthly quota"),
2228            "billing_cycle_exhausted"
2229        );
2230    }
2231
2232    #[test]
2233    fn classify_run_outcome_routed_is_success() {
2234        // "routed" is a canonical non-success status (task will be re-dispatched).
2235        // It should be classified as "success" so the audit trail distinguishes it
2236        // from genuine failures (#2801).
2237        let parse_result = Ok(ok_parse_result("routed"));
2238        assert_eq!(
2239            classify_run_outcome("routed", &parse_result, false),
2240            "success"
2241        );
2242    }
2243
2244    #[test]
2245    fn classify_run_outcome_blocked_is_blocked() {
2246        let parse_result = Ok(ok_parse_result("blocked"));
2247        assert_eq!(
2248            classify_run_outcome("blocked", &parse_result, false),
2249            "blocked"
2250        );
2251    }
2252
2253    #[test]
2254    fn classify_run_outcome_non_canonical_is_failed() {
2255        // Non-canonical statuses should be classified as "failed" in the audit.
2256        // The explicit error message is provided by build_run_audit.
2257        let parse_result = Ok(ok_parse_result("fix_deployed"));
2258        assert_eq!(
2259            classify_run_outcome("fix_deployed", &parse_result, false),
2260            "failed"
2261        );
2262    }
2263
2264    #[test]
2265    fn classify_run_outcome_kimi_billing_cycle_is_not_rate_limit() {
2266        // Regression test for issue #3529: Kimi's exact billing-cycle exhaustion
2267        // message was stored as outcome="rate_limit" because the generic "usage
2268        // limit" pattern in detect_rate_limit() matched before credit-exhaustion
2269        // detection ever ran. The outcome must reflect the actual failure mode.
2270        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2271            Err(agents::AgentError::RateLimit {
2272                message: "Failed to authenticate. API Error: 403 You've reached your usage \
2273                          limit for this billing cycle. Your quota will be refreshed soon."
2274                    .to_string(),
2275            });
2276        assert_eq!(
2277            classify_run_outcome("needs_review", &parse_result, false),
2278            "billing_cycle_exhausted"
2279        );
2280    }
2281
2282    #[test]
2283    fn classify_run_outcome_generic_rate_limit_is_unaffected() {
2284        // A genuine rate limit (no billing-cycle/credit-exhaustion markers) must
2285        // still classify as "rate_limit".
2286        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2287            Err(agents::AgentError::RateLimit {
2288                message: "429 Too Many Requests".to_string(),
2289            });
2290        assert_eq!(
2291            classify_run_outcome("needs_review", &parse_result, false),
2292            "rate_limit"
2293        );
2294    }
2295
2296    #[test]
2297    fn classify_run_outcome_auth_billing_cycle_is_not_failed() {
2298        // Auth-classified billing-cycle exhaustion (e.g. "billing cycle exhausted"
2299        // matches detect_auth_error's pattern list) must also be distinguished from
2300        // a generic auth failure.
2301        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2302            Err(agents::AgentError::Auth {
2303                message: "billing cycle exhausted for this account".to_string(),
2304            });
2305        assert_eq!(
2306            classify_run_outcome("needs_review", &parse_result, false),
2307            "billing_cycle_exhausted"
2308        );
2309    }
2310
2311    #[test]
2312    fn classify_run_outcome_generic_auth_error_is_unaffected() {
2313        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2314            Err(agents::AgentError::Auth {
2315                message: "401 Unauthorized".to_string(),
2316            });
2317        assert_eq!(
2318            classify_run_outcome("needs_review", &parse_result, false),
2319            "failed"
2320        );
2321    }
2322
2323    #[test]
2324    fn classify_run_outcome_interrupted() {
2325        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2326            Err(agents::AgentError::Interrupted {
2327                message: "exited without a result envelope".to_string(),
2328            });
2329        assert_eq!(
2330            classify_run_outcome("new", &parse_result, false),
2331            "interrupted"
2332        );
2333    }
2334
2335    #[tokio::test]
2336    async fn build_run_audit_success_ignores_stale_last_error() {
2337        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2338        let task_id = store
2339            .create(&NewTask {
2340                repo: "owner/repo".to_string(),
2341                origin: "internal".to_string(),
2342                title: "Test".to_string(),
2343                ..Default::default()
2344            })
2345            .await
2346            .unwrap();
2347        store
2348            .set_fields(
2349                task_id,
2350                &[("last_error", serde_json::json!("no error info available"))],
2351            )
2352            .await
2353            .unwrap();
2354
2355        let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2356        let parse_result = Ok(ok_parse_result("done"));
2357        let started_at = Utc::now();
2358        let audit = runner
2359            .build_run_audit(RunAuditInput {
2360                task_id: &task_id.to_string(),
2361                status: "done",
2362                parse_result: &parse_result,
2363                raw_stdout: "",
2364                raw_stderr: "",
2365                started_at: &started_at,
2366                error_override: None,
2367                elapsed_secs: Some(1),
2368                push_failed: false,
2369            })
2370            .await;
2371
2372        assert_eq!(audit.outcome, "success");
2373        assert_eq!(audit.error, "");
2374    }
2375
2376    #[tokio::test]
2377    async fn reroute_clear_wipes_agent_when_it_still_matches_the_run() {
2378        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2379        let store_id = store
2380            .create(&NewTask {
2381                external_id: Some("42".to_string()),
2382                repo: "owner/repo".to_string(),
2383                origin: "github".to_string(),
2384                title: "Test".to_string(),
2385                ..Default::default()
2386            })
2387            .await
2388            .unwrap();
2389        store
2390            .update_status(store_id, TaskStatus::InProgress)
2391            .await
2392            .unwrap();
2393        store
2394            .set_fields(
2395                store_id,
2396                &[
2397                    ("agent", serde_json::json!("claude")),
2398                    ("model", serde_json::json!("sonnet")),
2399                ],
2400            )
2401            .await
2402            .unwrap();
2403
2404        let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store));
2405        runner
2406            .clear_agent_on_reroute_unless_fallback("42", "claude")
2407            .await;
2408
2409        let task = store.get(store_id).await.unwrap();
2410        assert!(
2411            task.agent.as_deref().unwrap_or("").is_empty(),
2412            "matching in-progress agent must be cleared for a normal reroute, got {:?}",
2413            task.agent
2414        );
2415        assert!(
2416            task.model.as_deref().unwrap_or("").is_empty(),
2417            "model must be cleared with the agent, got {:?}",
2418            task.model
2419        );
2420    }
2421
2422    #[tokio::test]
2423    async fn reroute_clear_keeps_a_different_in_progress_fallback() {
2424        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2425        let store_id = store
2426            .create(&NewTask {
2427                external_id: Some("42".to_string()),
2428                repo: "owner/repo".to_string(),
2429                origin: "github".to_string(),
2430                title: "Test".to_string(),
2431                ..Default::default()
2432            })
2433            .await
2434            .unwrap();
2435        store
2436            .update_status(store_id, TaskStatus::InProgress)
2437            .await
2438            .unwrap();
2439        store
2440            .set_fields(
2441                store_id,
2442                &[
2443                    ("agent", serde_json::json!("codex")),
2444                    ("model", serde_json::json!("")),
2445                ],
2446            )
2447            .await
2448            .unwrap();
2449
2450        let runner = TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store));
2451        runner
2452            .clear_agent_on_reroute_unless_fallback("42", "claude")
2453            .await;
2454
2455        let task = store.get(store_id).await.unwrap();
2456        assert_eq!(task.agent.as_deref(), Some("codex"));
2457        assert!(
2458            task.model.as_deref().unwrap_or("").is_empty(),
2459            "model stays empty so routing picks one for the fallback, got {:?}",
2460            task.model
2461        );
2462    }
2463
2464    #[tokio::test]
2465    async fn build_run_audit_ok_parse_prefers_parsed_error_over_stale_last_error() {
2466        // needs_review classifies as success and wipes the error. The #3851
2467        // misattribution is an Ok parse whose outcome is failed or blocked
2468        let cases = [("needs_action", "failed"), ("blocked", "blocked")];
2469        let parsed_error = "passbox Touch ID prompt timed out for cf/read-token";
2470        let stale = "minimax auth error: API Error: 402 insufficient balance (1008), clearing agent/model for re-route";
2471        for (status, expected_outcome) in cases {
2472            let store = Arc::new(TaskStore::open_memory().await.unwrap());
2473            let task_id = store
2474                .create(&NewTask {
2475                    repo: "owner/repo".to_string(),
2476                    origin: "internal".to_string(),
2477                    title: "Test".to_string(),
2478                    ..Default::default()
2479                })
2480                .await
2481                .unwrap();
2482            store
2483                .set_fields(task_id, &[("last_error", serde_json::json!(stale))])
2484                .await
2485                .unwrap();
2486
2487            let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2488            let parse_result = Ok(agents::ParsedResponse {
2489                response: AgentResponse {
2490                    status: status.to_string(),
2491                    summary: "cache_stats.py did not run".to_string(),
2492                    accomplished: vec![],
2493                    remaining: vec![],
2494                    files: vec![],
2495                    error: Some(parsed_error.to_string()),
2496                    input_tokens: None,
2497                    output_tokens: None,
2498                    learnings: vec![],
2499                    delegations: vec![],
2500                    screenshots: vec![],
2501                    screenshot_note: None,
2502                },
2503                input_tokens: None,
2504                output_tokens: None,
2505                cost_usd: None,
2506                duration_ms: None,
2507            });
2508            let started_at = Utc::now();
2509            let audit = runner
2510                .build_run_audit(RunAuditInput {
2511                    task_id: &task_id.to_string(),
2512                    status,
2513                    parse_result: &parse_result,
2514                    raw_stdout: "",
2515                    raw_stderr: "",
2516                    started_at: &started_at,
2517                    error_override: None,
2518                    elapsed_secs: Some(261),
2519                    push_failed: false,
2520                })
2521                .await;
2522
2523            assert_eq!(audit.outcome, expected_outcome, "status={status}");
2524            assert_eq!(audit.error, parsed_error, "status={status}");
2525        }
2526    }
2527
2528    #[tokio::test]
2529    async fn build_run_audit_non_success_uses_fallback_error_when_empty() {
2530        let runner = TaskRunner::new("owner/repo".to_string());
2531        let parse_result = Ok(ok_parse_result("new"));
2532        let started_at = Utc::now();
2533        let audit = runner
2534            .build_run_audit(RunAuditInput {
2535                task_id: "1",
2536                status: "new",
2537                parse_result: &parse_result,
2538                raw_stdout: "",
2539                raw_stderr: "",
2540                started_at: &started_at,
2541                error_override: None,
2542                elapsed_secs: Some(1),
2543                push_failed: false,
2544            })
2545            .await;
2546
2547        assert_eq!(audit.outcome, "failed");
2548        // "new" is a canonical non-success status — no error message needed.
2549        assert_eq!(audit.error, "");
2550    }
2551
2552    #[tokio::test]
2553    async fn build_run_audit_non_canonical_status_reports_explicit_error() {
2554        // Non-canonical statuses (not in the allowlist) with no parse error
2555        // should get an explicit "unrecognized status: X" message instead of
2556        // the opaque "no error info available" placeholder (#2801).
2557        let runner = TaskRunner::new("owner/repo".to_string());
2558        let parse_result = Ok(ok_parse_result("fix_deployed"));
2559        let started_at = Utc::now();
2560        let audit = runner
2561            .build_run_audit(RunAuditInput {
2562                task_id: "1",
2563                status: "fix_deployed",
2564                parse_result: &parse_result,
2565                raw_stdout: "",
2566                raw_stderr: "",
2567                started_at: &started_at,
2568                error_override: None,
2569                elapsed_secs: Some(1),
2570                push_failed: false,
2571            })
2572            .await;
2573
2574        assert_eq!(audit.outcome, "failed");
2575        assert_eq!(audit.error, "unrecognized status: fix_deployed");
2576    }
2577
2578    #[tokio::test]
2579    async fn build_run_audit_canonical_non_success_status_no_error_message() {
2580        // Canonical non-success statuses (e.g. "routed") should not produce
2581        // an error message — they're expected outcomes, not failures.
2582        for status in &["routed"] {
2583            let runner = TaskRunner::new("owner/repo".to_string());
2584            let parse_result = Ok(ok_parse_result(status));
2585            let started_at = Utc::now();
2586            let audit = runner
2587                .build_run_audit(RunAuditInput {
2588                    task_id: "1",
2589                    status,
2590                    parse_result: &parse_result,
2591                    raw_stdout: "",
2592                    raw_stderr: "",
2593                    started_at: &started_at,
2594                    error_override: None,
2595                    elapsed_secs: Some(1),
2596                    push_failed: false,
2597                })
2598                .await;
2599
2600            assert_eq!(audit.outcome, "success", "status={status}");
2601            assert_eq!(
2602                audit.error, "",
2603                "status={status}: no error for canonical status"
2604            );
2605        }
2606    }
2607
2608    #[tokio::test]
2609    async fn build_run_audit_blocked_uses_summary_when_error_missing() {
2610        let runner = TaskRunner::new("owner/repo".to_string());
2611        let parse_result = Ok(agents::ParsedResponse {
2612            response: AgentResponse {
2613                status: "blocked".to_string(),
2614                summary: "Could not proceed due to missing credentials".to_string(),
2615                accomplished: vec![],
2616                remaining: vec![],
2617                files: vec![],
2618                error: None,
2619                input_tokens: None,
2620                output_tokens: None,
2621                learnings: vec![],
2622                delegations: vec![],
2623                screenshots: vec![],
2624                screenshot_note: None,
2625            },
2626            input_tokens: None,
2627            output_tokens: None,
2628            cost_usd: None,
2629            duration_ms: None,
2630        });
2631        let started_at = Utc::now();
2632        let audit = runner
2633            .build_run_audit(RunAuditInput {
2634                task_id: "1",
2635                status: "blocked",
2636                parse_result: &parse_result,
2637                raw_stdout: "",
2638                raw_stderr: "",
2639                started_at: &started_at,
2640                error_override: None,
2641                elapsed_secs: Some(1),
2642                push_failed: false,
2643            })
2644            .await;
2645
2646        assert_eq!(audit.outcome, "blocked");
2647        assert_eq!(audit.error, "Could not proceed due to missing credentials");
2648    }
2649
2650    // ── resolve_project_dir ──────────────────────────────────────────────────
2651
2652    #[tokio::test]
2653    async fn resolve_project_dir_uses_project_dir_env() {
2654        // When PROJECT_DIR env var is set, it should always win.
2655        let runner = TaskRunner::new("owner/repo".to_string());
2656
2657        // Use a temp dir so the path exists
2658        let dir = tempfile::tempdir().unwrap();
2659        let dir_str = dir.path().to_string_lossy().to_string();
2660
2661        // Temporarily set the env var — note: tests run in parallel so we use a
2662        // dedicated key that only this test touches. std::env::set_var is not
2663        // thread-safe in general, but the subsequent call is synchronous and
2664        // isolated to the temp dir created above.
2665        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2666        unsafe { env::set_var("PROJECT_DIR", &dir_str) };
2667        let result = runner.resolve_project_dir().await;
2668        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2669        unsafe { env::remove_var("PROJECT_DIR") };
2670
2671        assert!(
2672            result.is_ok(),
2673            "resolve_project_dir should succeed with PROJECT_DIR set"
2674        );
2675        assert_eq!(result.unwrap(), dir.path());
2676    }
2677
2678    #[tokio::test]
2679    async fn resolve_project_dir_empty_project_dir_env_falls_through() {
2680        // An empty PROJECT_DIR should be ignored and fall through to other logic.
2681        let runner = TaskRunner::new("owner/testrepo-nonexistent".to_string());
2682        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2683        unsafe { env::set_var("PROJECT_DIR", "") };
2684        let result = runner.resolve_project_dir().await;
2685        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2686        unsafe { env::remove_var("PROJECT_DIR") };
2687
2688        // No active config entry, no legacy project_dir, no bare clone for this
2689        // repo: should return an explicit error rather than falling back to the
2690        // service's own CWD (see issue #3416).
2691        assert!(
2692            result.is_err(),
2693            "resolve_project_dir should error when no project dir can be resolved"
2694        );
2695        let err = result.unwrap_err().to_string();
2696        assert!(
2697            err.contains("owner/testrepo-nonexistent"),
2698            "error should name the unresolvable repo: {err}"
2699        );
2700    }
2701
2702    // ── mock backend for process_delegations ─────────────────────────────────
2703
2704    struct TrackingBackend {
2705        sub_tasks_created: Arc<Mutex<Vec<(String, String)>>>, // (title, parent_id)
2706        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
2707        comments: Arc<Mutex<Vec<(String, String)>>>, // (id, body)
2708    }
2709
2710    impl TrackingBackend {
2711        fn new() -> Self {
2712            Self {
2713                sub_tasks_created: Arc::new(Mutex::new(vec![])),
2714                status_updates: Arc::new(Mutex::new(vec![])),
2715                comments: Arc::new(Mutex::new(vec![])),
2716            }
2717        }
2718    }
2719
2720    #[async_trait]
2721    impl ExternalBackend for TrackingBackend {
2722        fn name(&self) -> &'static str {
2723            "tracking"
2724        }
2725        async fn create_task(
2726            &self,
2727            _t: &str,
2728            _b: &str,
2729            _l: &[String],
2730        ) -> anyhow::Result<ExternalId> {
2731            Ok(ExternalId("new".to_string()))
2732        }
2733        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2734            Ok(ExternalTask {
2735                id: id.clone(),
2736                title: String::new(),
2737                body: String::new(),
2738                state: "open".to_string(),
2739                labels: vec![],
2740                author: String::new(),
2741                created_at: "2026-01-01T00:00:00Z".to_string(),
2742                updated_at: "2026-01-01T00:00:00Z".to_string(),
2743                url: String::new(),
2744            })
2745        }
2746        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
2747            Ok(vec![])
2748        }
2749        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2750            Ok(vec![])
2751        }
2752        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2753            self.comments
2754                .lock()
2755                .unwrap()
2756                .push((id.0.clone(), body.to_string()));
2757            Ok(())
2758        }
2759        async fn set_labels(&self, _id: &ExternalId, _l: &[String]) -> anyhow::Result<()> {
2760            Ok(())
2761        }
2762        async fn remove_label(&self, _id: &ExternalId, _l: &str) -> anyhow::Result<()> {
2763            Ok(())
2764        }
2765        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2766            Ok(vec![])
2767        }
2768        async fn create_sub_task(
2769            &self,
2770            parent: &ExternalId,
2771            title: &str,
2772            _body: &str,
2773            _l: &[String],
2774        ) -> anyhow::Result<ExternalId> {
2775            self.sub_tasks_created
2776                .lock()
2777                .unwrap()
2778                .push((title.to_string(), parent.0.clone()));
2779            Ok(ExternalId(format!("child-{title}")))
2780        }
2781        async fn ensure_status_label(&self, _l: &str) -> anyhow::Result<()> {
2782            Ok(())
2783        }
2784        async fn has_open_issue_with_title(&self, _t: &str, _l: &str) -> anyhow::Result<bool> {
2785            Ok(false)
2786        }
2787        async fn health_check(&self) -> anyhow::Result<()> {
2788            Ok(())
2789        }
2790        async fn is_pr_merged(&self, _b: &str) -> anyhow::Result<bool> {
2791            Ok(false)
2792        }
2793        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2794            Ok(None)
2795        }
2796        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
2797            Ok(vec![])
2798        }
2799        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2800            self.status_updates
2801                .lock()
2802                .unwrap()
2803                .push((id.0.clone(), status));
2804            Ok(())
2805        }
2806    }
2807
2808    fn make_task(id: &str) -> ExternalTask {
2809        ExternalTask {
2810            id: ExternalId(id.to_string()),
2811            title: format!("Task {id}"),
2812            body: String::new(),
2813            state: "open".to_string(),
2814            labels: vec![],
2815            author: "bot".to_string(),
2816            created_at: "2026-01-01T00:00:00Z".to_string(),
2817            updated_at: "2026-01-01T00:00:00Z".to_string(),
2818            url: String::new(),
2819        }
2820    }
2821
2822    // ── process_delegations ───────────────────────────────────────────────────
2823
2824    #[expect(
2825        clippy::await_holding_lock,
2826        reason = "the env lock must stay held across the awaited calls"
2827    )]
2828    #[tokio::test]
2829    async fn process_delegations_creates_subtasks_and_blocks_parent() {
2830        let _guard = ENV_LOCK.lock().unwrap();
2831        let temp_home = TempDir::new().unwrap();
2832        let orch_home = temp_home.path().join(".orch");
2833        fs::create_dir_all(&orch_home).await.unwrap();
2834        let old_orch_home = env::var("ORCH_HOME").ok();
2835        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2836        unsafe { env::set_var("ORCH_HOME", &orch_home) };
2837
2838        let runner = TaskRunner::new("owner/repo".to_string());
2839        let parent = make_task("99");
2840        let backend = Arc::new(TrackingBackend::new());
2841        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<TrackingBackend>::clone(&backend);
2842
2843        let delegations = vec![
2844            Delegation {
2845                title: "Subtask A".to_string(),
2846                body: "Do A".to_string(),
2847                labels: vec!["enhancement".to_string()],
2848            },
2849            Delegation {
2850                title: "Subtask B".to_string(),
2851                body: "Do B".to_string(),
2852                labels: vec![],
2853            },
2854        ];
2855
2856        runner
2857            .process_delegations(&parent, &delegations, &backend_dyn)
2858            .await
2859            .unwrap();
2860
2861        // Two sub-tasks should have been created
2862        let created = backend.sub_tasks_created.lock().unwrap();
2863        assert_eq!(created.len(), 2);
2864        assert_eq!(created[0].1, "99", "parent id should be 99");
2865        assert!(
2866            created.iter().any(|(t, _)| t == "Subtask A"),
2867            "Subtask A should be created"
2868        );
2869        assert!(
2870            created.iter().any(|(t, _)| t == "Subtask B"),
2871            "Subtask B should be created"
2872        );
2873        drop(created);
2874
2875        // Parent should be marked blocked
2876        let updates = backend.status_updates.lock().unwrap();
2877        assert!(
2878            updates
2879                .iter()
2880                .any(|(id, s)| id == "99" && *s == Status::Blocked),
2881            "parent should be blocked"
2882        );
2883        drop(updates);
2884
2885        // A comment summarising the delegations should be posted
2886        let comments = backend.comments.lock().unwrap();
2887        assert!(
2888            !comments.is_empty(),
2889            "a delegation summary comment should be posted"
2890        );
2891        let comment_body = &comments[0].1;
2892        assert!(
2893            comment_body.contains("Subtask A"),
2894            "summary should mention Subtask A"
2895        );
2896        assert!(
2897            comment_body.contains("Subtask B"),
2898            "summary should mention Subtask B"
2899        );
2900
2901        if let Some(old) = old_orch_home {
2902            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2903            unsafe { env::set_var("ORCH_HOME", old) };
2904        } else {
2905            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2906            unsafe { env::remove_var("ORCH_HOME") };
2907        }
2908    }
2909
2910    #[expect(
2911        clippy::await_holding_lock,
2912        reason = "the env lock must stay held across the awaited calls"
2913    )]
2914    #[tokio::test]
2915    async fn process_delegations_single_subtask() {
2916        let _guard = ENV_LOCK.lock().unwrap();
2917        let temp_home = TempDir::new().unwrap();
2918        let orch_home = temp_home.path().join(".orch");
2919        fs::create_dir_all(&orch_home).await.unwrap();
2920        let old_orch_home = env::var("ORCH_HOME").ok();
2921        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2922        unsafe { env::set_var("ORCH_HOME", &orch_home) };
2923
2924        let runner = TaskRunner::new("owner/repo".to_string());
2925        let parent = make_task("101");
2926        let backend = Arc::new(TrackingBackend::new());
2927        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<TrackingBackend>::clone(&backend);
2928
2929        let delegations = vec![Delegation {
2930            title: "Only Child".to_string(),
2931            body: "Do only child".to_string(),
2932            labels: vec!["feature".to_string()],
2933        }];
2934
2935        runner
2936            .process_delegations(&parent, &delegations, &backend_dyn)
2937            .await
2938            .unwrap();
2939
2940        let created = backend.sub_tasks_created.lock().unwrap();
2941        assert_eq!(created.len(), 1);
2942        assert_eq!(created[0].0, "Only Child");
2943        drop(created);
2944
2945        let updates = backend.status_updates.lock().unwrap();
2946        assert!(
2947            updates
2948                .iter()
2949                .any(|(id, s)| id == "101" && *s == Status::Blocked),
2950            "parent should be blocked"
2951        );
2952        drop(updates);
2953
2954        let comments = backend.comments.lock().unwrap();
2955        let body = &comments[0].1;
2956        assert!(
2957            body.contains("1 subtask"),
2958            "comment should count one subtask"
2959        );
2960
2961        if let Some(old) = old_orch_home {
2962            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2963            unsafe { env::set_var("ORCH_HOME", old) };
2964        } else {
2965            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
2966            unsafe { env::remove_var("ORCH_HOME") };
2967        }
2968    }
2969
2970    // ── weight signal logic ───────────────────────────────────────────────────
2971
2972    /// Test helper that mirrors the weight signal logic from `run_with_context`.
2973    /// Reroutes only emit `RateLimited` when the error type is genuinely "rate_limit".
2974    fn weight_signal_for(status: &str, error_type: &str, agent: &str) -> WeightSignal {
2975        let is_rerouted = status == "new" || status == "routed";
2976        let is_rate_limit_error = error_type == "rate_limit";
2977        if is_rerouted {
2978            if is_rate_limit_error {
2979                WeightSignal::RateLimited {
2980                    agent: agent.to_string(),
2981                }
2982            } else {
2983                WeightSignal::Rerouted
2984            }
2985        } else if status == "done"
2986            || status == "needs_review"
2987            || status == "in_progress"
2988            || status == "in_review"
2989        {
2990            WeightSignal::Success {
2991                agent: agent.to_string(),
2992            }
2993        } else if status == "blocked" {
2994            WeightSignal::Blocked
2995        } else {
2996            WeightSignal::None
2997        }
2998    }
2999
3000    #[test]
3001    fn weight_signal_success_includes_needs_review() {
3002        // The normal happy-path for code tasks: agent reports done, has a PR,
3003        // so handle_success returns "needs_review". This must map to Success.
3004        let signal = weight_signal_for("needs_review", "success", "claude");
3005        assert!(
3006            matches!(signal, WeightSignal::Success { agent } if agent == "claude"),
3007            "needs_review should produce WeightSignal::Success"
3008        );
3009    }
3010
3011    #[test]
3012    fn needs_review_signal_success_for_empty_error() {
3013        let signal = needs_review_signal("", "claude");
3014        assert!(matches!(signal, WeightSignal::Success { agent } if agent == "claude"));
3015    }
3016
3017    #[test]
3018    fn needs_review_signal_failed_after_timeout_with_no_fallback() {
3019        let signal =
3020            needs_review_signal("claude timed out after 1804s, no fallback agents", "claude");
3021        assert!(matches!(signal, WeightSignal::Failed { agent } if agent == "claude"));
3022    }
3023
3024    #[test]
3025    fn needs_review_signal_rate_limited_for_rate_limit() {
3026        let signal = needs_review_signal("rate limit exceeded, try again later", "claude");
3027        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3028    }
3029
3030    #[test]
3031    fn weight_signal_success_for_all_done_statuses() {
3032        for status in ["done", "needs_review", "in_progress", "in_review"] {
3033            let signal = weight_signal_for(status, "success", "codex");
3034            assert!(
3035                matches!(signal, WeightSignal::Success { agent } if agent == "codex"),
3036                "{status} should produce WeightSignal::Success"
3037            );
3038        }
3039    }
3040
3041    #[test]
3042    fn weight_signal_rate_limited_for_reroute_with_rate_limit_error() {
3043        // Reroute due to rate limit should emit RateLimited.
3044        let signal = weight_signal_for("new", "rate_limit", "claude");
3045        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3046    }
3047
3048    #[test]
3049    fn weight_signal_rate_limited_for_routed_status_with_rate_limit_error() {
3050        let signal = weight_signal_for("routed", "rate_limit", "claude");
3051        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
3052    }
3053
3054    #[test]
3055    fn weight_signal_rerouted_for_non_rate_limit_reroute() {
3056        // Reroute due to silence detection, timeout, or other non-rate-limit errors
3057        // should emit Rerouted (not RateLimited and not None).
3058        // Rerouted maps to "new" in tick so the task stays retryable and is never
3059        // converted to needs_review → done via the no-code review shortcut.
3060        for error_type in ["timeout", "silence detected", "parse_error", "failed"] {
3061            for status in ["new", "routed"] {
3062                let signal = weight_signal_for(status, error_type, "claude");
3063                assert!(
3064                    matches!(signal, WeightSignal::Rerouted),
3065                    "{status} with error_type={error_type} should produce WeightSignal::Rerouted, not RateLimited or None"
3066                );
3067            }
3068        }
3069    }
3070
3071    #[test]
3072    fn silence_reset_rerouted_never_becomes_needs_review() {
3073        // Regression test for: silence detection sets task to new, runner returns
3074        // WeightSignal::Rerouted, tick must map that to "new" — not "needs_review".
3075        // If it mapped to needs_review the no-code shortcut would close the task as
3076        // done with no successful agent run in the audit trail.
3077        let signal = weight_signal_for("new", "silence detected", "claude");
3078        assert!(
3079            matches!(signal, WeightSignal::Rerouted),
3080            "silence-reset reroute must produce Rerouted, never None/needs_review"
3081        );
3082        // Verify tick mapping: Rerouted => "new" (not needs_review or done)
3083        let display_status = match &signal {
3084            WeightSignal::Rerouted => "new",
3085            WeightSignal::None => "needs_review", // the old (buggy) path
3086            _ => "other",
3087        };
3088        assert_eq!(
3089            display_status, "new",
3090            "Rerouted must map to 'new' in tick, not 'needs_review'"
3091        );
3092    }
3093
3094    #[test]
3095    fn weight_signal_blocked_status() {
3096        let signal = weight_signal_for("blocked", "success", "claude");
3097        assert!(matches!(signal, WeightSignal::Blocked));
3098    }
3099
3100    // ── parse_success_output token extraction ────────────────────────────────
3101
3102    /// Mock runner that always returns InvalidResponse with truncated text,
3103    /// simulating what minimax/codex do when they can't parse structured JSON.
3104    struct FailingMockRunner;
3105
3106    impl agents::AgentRunner for FailingMockRunner {
3107        #[cfg(test)]
3108        fn name(&self) -> &'static str {
3109            "opencode"
3110        }
3111
3112        fn stream_events(&self, _event: &serde_json::Value) -> Vec<agents::StreamEvent> {
3113            vec![]
3114        }
3115
3116        fn build_command(
3117            &self,
3118            _model: Option<&str>,
3119            _timeout_cmd: &str,
3120            _sys_file: &str,
3121            _msg_file: &str,
3122            _permissions: &agents::PermissionRules,
3123        ) -> String {
3124            String::new()
3125        }
3126
3127        fn parse_response(&self, raw: &str) -> Result<agents::ParsedResponse, agents::AgentError> {
3128            // Simulate what minimax does: truncate to 300 chars
3129            Err(agents::AgentError::InvalidResponse {
3130                raw: raw.chars().take(30).collect(),
3131            })
3132        }
3133
3134        fn extract_text(&self, raw: &str) -> Result<String, agents::AgentError> {
3135            Err(agents::AgentError::InvalidResponse {
3136                raw: raw.to_string(),
3137            })
3138        }
3139
3140        fn classify_error(
3141            &self,
3142            exit_code: i32,
3143            _stdout: &str,
3144            _stderr: &str,
3145        ) -> agents::AgentError {
3146            agents::AgentError::Unknown {
3147                exit_code,
3148                message: "mock error".into(),
3149            }
3150        }
3151
3152        fn router_command(&self, _prompt: &str, _model: Option<&str>) -> anyhow::Result<Command> {
3153            anyhow::bail!("not implemented for mock")
3154        }
3155    }
3156
3157    #[test]
3158    fn parse_success_output_extracts_tokens_from_raw_stdout_not_error_raw() {
3159        // OpenCode NDJSON output with token metadata in step_finish
3160        let ndjson = r#"{"type":"text","timestamp":1,"part":{"type":"text","text":"Here is my response: {\"status\":\"done\",\"summary\":\"fixed\"}}"}}
3161{"type":"step_finish","timestamp":2,"part":{"type":"step-finish","reason":"stop","cost":0,"tokens":{"total":95177,"input":94920,"output":257}}}"#;
3162
3163        let runner = FailingMockRunner;
3164        let result = parse_success_output("1550", "opencode", &runner, ndjson);
3165
3166        // Should succeed (synthesized from text) with tokens preserved
3167        let parsed = result.expect("should synthesize response from text");
3168        assert_eq!(parsed.input_tokens, Some(94920));
3169        assert_eq!(parsed.output_tokens, Some(257));
3170    }
3171
3172    #[test]
3173    fn parse_success_output_no_tokens_when_ndjson_has_none() {
3174        // Plain text with an explicit completion signal but no token metadata.
3175        // Uses "No open positions" which is in the explicit_done list.
3176        let text = "No open positions, no trade executions, no conditions change.";
3177
3178        let runner = FailingMockRunner;
3179        let result = parse_success_output("1551", "opencode", &runner, text);
3180
3181        // Should synthesize a "done" response, but without tokens (no NDJSON envelope).
3182        let parsed = result.expect("should synthesize response from explicit-done text");
3183        assert_eq!(parsed.input_tokens, None);
3184        assert_eq!(parsed.output_tokens, None);
3185    }
3186
3187    #[test]
3188    fn parse_success_output_uses_agent_result_directly() {
3189        // Here we test that if find_agent_result succeeds and returns valid JSON, we use it directly
3190        // WITHOUT calling agent_runner.parse_response at all.
3191        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"status\":\"done\",\"summary\":\"my summary\",\"accomplished\":[],\"remaining\":[],\"files\":[]}"}"#;
3192
3193        // FailingMockRunner always returns InvalidResponse, so if we called it, we'd synthesize "done" instead of getting "my summary".
3194        let runner = FailingMockRunner;
3195        let result = parse_success_output("999", "claude", &runner, ndjson);
3196
3197        let parsed = result.expect("should parse successfully via find_agent_result");
3198        assert_eq!(parsed.response.status, "done");
3199        assert_eq!(parsed.response.summary, "my summary");
3200    }
3201
3202    #[test]
3203    fn parse_success_output_treats_agent_result_is_error_as_authoritative() {
3204        // Here we test that if find_agent_result returns is_error=true, we return AgentFailed
3205        // instead of falling back to synthesis or agent_runner.parse_response.
3206        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"I failed because of reasons"}"#;
3207
3208        let runner = FailingMockRunner;
3209        let result = parse_success_output("999", "claude", &runner, ndjson);
3210
3211        let err = result.expect_err("should return error");
3212        match err {
3213            agents::AgentError::AgentFailed { message } => {
3214                assert_eq!(message, "I failed because of reasons");
3215            }
3216            _ => panic!("expected AgentFailed, got {err:?}"),
3217        }
3218    }
3219
3220    #[test]
3221    fn parse_success_output_classifies_model_unavailable_from_is_error_result() {
3222        // Regression test for issue #3607: the is_error=true NDJSON path
3223        // (used by codex's find_codex_result, which stringifies its own
3224        // classified AgentError into result_text) must re-classify a
3225        // model-not-found message as ModelUnavailable so it gets
3226        // model-scoped cooldown treatment instead of degrading to a
3227        // generic AgentFailed, which triggers an agent-wide cooldown.
3228        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"model unavailable (gpt-5.5): Reconnecting... 2/5 (unexpected status 404 Not Found: The model `gpt-5.5` does not exist or you do not have access to it.)"}"#;
3229
3230        let runner = FailingMockRunner;
3231        let result = parse_success_output("999", "claude", &runner, ndjson);
3232
3233        let err = result.expect_err("should return error");
3234        match err {
3235            agents::AgentError::ModelUnavailable { model, .. } => {
3236                assert_eq!(model, "gpt-5.5");
3237            }
3238            _ => panic!("expected ModelUnavailable, got {err:?}"),
3239        }
3240    }
3241
3242    #[test]
3243    fn nonzero_exit_with_ndjson_success_is_parsed_as_success() {
3244        // Simulate an agent that exits with non-zero code but emits NDJSON result
3245        // with is_error=false and a JSON result payload. The runner should accept
3246        // this as success because the NDJSON result is authoritative.
3247        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1234,"result":"{\"status\":\"done\",\"summary\":\"All good\"}","usage":{"input_tokens":10,"output_tokens":5}}"#;
3248
3249        // Use a runner that would normally fail parsing to ensure we're using
3250        // the find_agent_result path. FailingMockRunner.parse_response returns
3251        // InvalidResponse, but parse_success_output should short-circuit.
3252        let runner = FailingMockRunner;
3253
3254        // Construct a fake SessionOutput
3255        let session_output = session::SessionOutput {
3256            exit_code: 1,
3257            raw_stdout: ndjson.to_string(),
3258            raw_stderr: String::new(),
3259            elapsed_secs: Some(1),
3260            sandbox_error: None,
3261        };
3262
3263        // Call the helper we added via the task runner logic by reusing the
3264        // parse_success_output directly since it encapsulates the same behavior.
3265        let parsed =
3266            parse_success_output("task-ndjson", "claude", &runner, &session_output.raw_stdout)
3267                .expect("should parse NDJSON success");
3268        assert_eq!(parsed.response.status, "done");
3269        assert_eq!(parsed.response.summary, "All good");
3270        assert_eq!(parsed.input_tokens, Some(10));
3271        assert_eq!(parsed.output_tokens, Some(5));
3272    }
3273
3274    // ── agent label update after reroute ─────────────────────────────────────
3275
3276    /// Backend that wraps TrackingBackend but can be configured to fail label ops.
3277    struct LabelFailingBackend {
3278        inner: TrackingBackend,
3279        /// If set, set_labels returns this error.
3280        set_labels_err: Option<String>,
3281    }
3282
3283    impl LabelFailingBackend {
3284        fn new() -> Self {
3285            Self {
3286                inner: TrackingBackend::new(),
3287                set_labels_err: None,
3288            }
3289        }
3290        fn with_set_labels_err(mut self, err: impl Into<String>) -> Self {
3291            self.set_labels_err = Some(err.into());
3292            self
3293        }
3294    }
3295
3296    impl Default for LabelFailingBackend {
3297        fn default() -> Self {
3298            Self::new()
3299        }
3300    }
3301
3302    #[async_trait]
3303    impl ExternalBackend for LabelFailingBackend {
3304        fn name(&self) -> &'static str {
3305            "label_failing"
3306        }
3307        async fn create_task(
3308            &self,
3309            _t: &str,
3310            _b: &str,
3311            _l: &[String],
3312        ) -> anyhow::Result<ExternalId> {
3313            Ok(ExternalId("new".to_string()))
3314        }
3315        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3316            self.inner.get_task(id).await
3317        }
3318        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
3319            Ok(vec![])
3320        }
3321        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3322            Ok(vec![])
3323        }
3324        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3325            self.inner.post_comment(id, body).await
3326        }
3327        async fn set_labels(&self, id: &ExternalId, labels: &[String]) -> anyhow::Result<()> {
3328            if let Some(ref err) = self.set_labels_err {
3329                return Err(anyhow::anyhow!("{err}"));
3330            }
3331            self.inner.set_labels(id, labels).await
3332        }
3333        async fn remove_label(&self, id: &ExternalId, label: &str) -> anyhow::Result<()> {
3334            self.inner.remove_label(id, label).await
3335        }
3336        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3337            Ok(vec![])
3338        }
3339        async fn create_sub_task(
3340            &self,
3341            parent: &ExternalId,
3342            title: &str,
3343            body: &str,
3344            labels: &[String],
3345        ) -> anyhow::Result<ExternalId> {
3346            self.inner
3347                .create_sub_task(parent, title, body, labels)
3348                .await
3349        }
3350        async fn ensure_status_label(&self, label: &str) -> anyhow::Result<()> {
3351            self.inner.ensure_status_label(label).await
3352        }
3353        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3354            self.inner.update_status(id, status).await
3355        }
3356        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3357            self.inner.get_authenticated_user().await
3358        }
3359        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
3360            Ok(vec![])
3361        }
3362        async fn health_check(&self) -> anyhow::Result<()> {
3363            self.inner.health_check().await
3364        }
3365    }
3366
3367    /// `update_agent_label_after_reroute` returns `false` when `set_labels` fails.
3368    ///
3369    /// This tests the extracted helper directly, bypassing the full `run_with_context`
3370    /// path (which would fail earlier on worktree setup in a unit-test environment).
3371    #[tokio::test]
3372    async fn update_agent_label_returns_false_on_set_labels_failure() {
3373        let task = ExternalTask {
3374            id: ExternalId("test-reroute-1".to_string()),
3375            title: "Test".to_string(),
3376            body: "body".to_string(),
3377            state: "open".to_string(),
3378            labels: vec![],
3379            author: "bot".to_string(),
3380            created_at: "2026-01-01T00:00:00Z".to_string(),
3381            updated_at: "2026-01-01T00:00:00Z".to_string(),
3382            url: String::new(),
3383        };
3384        let backend: Arc<dyn ExternalBackend> =
3385            Arc::new(LabelFailingBackend::new().with_set_labels_err("network error"));
3386
3387        let result =
3388            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
3389                .await;
3390
3391        assert!(
3392            !result,
3393            "update_agent_label_after_reroute should return false when set_labels fails"
3394        );
3395    }
3396
3397    /// `update_agent_label_after_reroute` returns `true` when `set_labels` succeeds.
3398    #[tokio::test]
3399    async fn update_agent_label_returns_true_on_success() {
3400        let task = ExternalTask {
3401            id: ExternalId("test-reroute-2".to_string()),
3402            title: "Test".to_string(),
3403            body: "body".to_string(),
3404            state: "open".to_string(),
3405            labels: vec![],
3406            author: "bot".to_string(),
3407            created_at: "2026-01-01T00:00:00Z".to_string(),
3408            updated_at: "2026-01-01T00:00:00Z".to_string(),
3409            url: String::new(),
3410        };
3411        // LabelFailingBackend with no injected error delegates to TrackingBackend (always Ok).
3412        let backend: Arc<dyn ExternalBackend> = Arc::new(LabelFailingBackend::new());
3413
3414        let result =
3415            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
3416                .await;
3417
3418        // ensure_label is skipped in #[cfg(test)], set_labels succeeds via mock.
3419        assert!(
3420            result,
3421            "update_agent_label_after_reroute should return true when set_labels succeeds"
3422        );
3423    }
3424
3425    // ── NDJSON envelope fallback (issue #3078) ────────────────────────────────
3426
3427    /// Pattern A: agent emits domain JSON as result (no `status` field).
3428    /// NDJSON envelope says is_error=false, subtype=success.
3429    /// Was previously returning InvalidResponse → task re-ran.
3430    #[test]
3431    fn parse_success_output_ndjson_envelope_fallback_domain_json_result() {
3432        let domain_result = r#"{"file_path":"md/journal/2026-05-08-morning-briefing.md","summary":"Morning briefing filed","health":{"status":"ok"},"priorities":[]}"#;
3433        let ndjson = format!(
3434            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{domain_result:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
3435        );
3436
3437        let runner = agents::get_runner("glm");
3438        let parsed = parse_success_output("149200", "glm", &*runner, &ndjson)
3439            .expect("should synthesize done from NDJSON envelope when result is domain JSON");
3440
3441        assert_eq!(parsed.response.status, "done");
3442        assert!(
3443            parsed.response.summary.contains("file_path") || !parsed.response.summary.is_empty(),
3444            "summary should be populated from domain JSON result text"
3445        );
3446        assert_eq!(parsed.input_tokens, Some(3000));
3447        assert_eq!(parsed.output_tokens, Some(400));
3448    }
3449
3450    /// Grok-style envelope: no terminal_reason, subtype=success.
3451    #[test]
3452    fn parse_success_output_envelope_fallback_without_terminal_reason() {
3453        let receipt = r#"{"ok":true,"report":"notes.md","commit":"abc"}"#;
3454        let ndjson = format!(
3455            r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","result":{receipt:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
3456        );
3457        let runner = agents::get_runner("grok");
3458        let parsed = parse_success_output("1", "grok", &*runner, &ndjson).expect("done");
3459        assert_eq!(parsed.response.status, "done");
3460        assert_eq!(parsed.input_tokens, Some(3000));
3461    }
3462
3463    /// Pattern B: agent emits plain prose as result, no completion keywords.
3464    /// NDJSON envelope says is_error=false, subtype=success.
3465    /// Was previously returning InvalidResponse → task re-ran.
3466    #[test]
3467    fn parse_success_output_ndjson_envelope_fallback_plain_prose_result() {
3468        let prose = "PR #1008 now has zero diff vs main. The reverted ledger changes are gone and the branch is clean.";
3469        let ndjson = format!(
3470            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{prose:?},"usage":{{"input_tokens":2500,"output_tokens":300}}}}"#
3471        );
3472
3473        let runner = agents::get_runner("claude");
3474        let parsed = parse_success_output("149164", "claude", &*runner, &ndjson)
3475            .expect("should synthesize done from NDJSON envelope when result is plain prose");
3476
3477        assert_eq!(parsed.response.status, "done");
3478        assert_eq!(parsed.response.summary, prose);
3479        assert_eq!(parsed.input_tokens, Some(2500));
3480        assert_eq!(parsed.output_tokens, Some(300));
3481    }
3482
3483    /// Verify that is_error=true in NDJSON envelope still returns AgentFailed,
3484    /// not "done" — the envelope fallback must not override explicit errors.
3485    #[test]
3486    fn parse_success_output_ndjson_envelope_fallback_skipped_when_is_error_true() {
3487        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"result":"Something went wrong with no status"}"#;
3488
3489        let runner = agents::get_runner("claude");
3490        let err = parse_success_output("149999", "claude", &*runner, ndjson)
3491            .expect_err("is_error=true should return AgentFailed");
3492
3493        assert!(matches!(err, agents::AgentError::AgentFailed { .. }));
3494    }
3495
3496    fn tool_pair(id: &str, name: &str, cmd: &str, is_error: bool) -> String {
3497        format!(
3498            "{}\n{}\n",
3499            serde_json::json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":id,"name":name,"input":{"command":cmd}}]}}),
3500            serde_json::json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":id,"is_error":is_error,"content":"x"}]}}),
3501        )
3502    }
3503
3504    #[test]
3505    fn salvage_wrapup_accepts_commit_then_cleanup() {
3506        let raw = format!(
3507            "{}{}",
3508            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3509            tool_pair("2", "run_terminal_cmd", "git status", false)
3510        );
3511        assert!(salvage_wrapup(&raw).is_some());
3512    }
3513
3514    #[test]
3515    fn salvage_wrapup_rejects_edit_after_commit_or_unanswered_call() {
3516        let edit = format!(
3517            "{}{}",
3518            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3519            tool_pair("2", "search_replace", "", false)
3520        );
3521        assert!(salvage_wrapup(&edit).is_none());
3522        let pending = format!(
3523            "{}{}",
3524            tool_pair("1", "run_terminal_cmd", "git commit -m x", false),
3525            serde_json::json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":"2","name":"run_terminal_cmd","input":{"command":"cargo test"}}]}})
3526        );
3527        assert!(salvage_wrapup(&pending).is_none());
3528        assert!(salvage_wrapup(&tool_pair("1", "run_terminal_cmd", "ls", false)).is_none());
3529    }
3530
3531    /// Grok writes `is_error: true` and `errors: ["cancelled"]` when the timeout
3532    /// wrapper kills it. Exit 124 is the time limit, so this is Timeout
3533    #[test]
3534    fn parse_session_output_exit124_cancelled_envelope_is_timeout() {
3535        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"stop_reason":"cancelled","errors":["cancelled"],"duration_ms":2697000}"#;
3536        let session = session::SessionOutput {
3537            exit_code: 124,
3538            raw_stdout: ndjson.to_string(),
3539            raw_stderr: String::new(),
3540            elapsed_secs: Some(2705),
3541            sandbox_error: None,
3542        };
3543
3544        let runner = agents::get_runner("grok");
3545        let err = parse_session_output("4068", "grok", &*runner, &session)
3546            .expect_err("cancelled at the time limit should be an error");
3547        match err {
3548            agents::AgentError::Timeout { elapsed } => {
3549                assert_eq!(elapsed.as_secs(), 2705);
3550            }
3551            _ => panic!("exit 124 with cancelled envelope should be Timeout, got {err:?}"),
3552        }
3553    }
3554
3555    /// A provider error with a normal non-zero exit is AgentFailed
3556    #[test]
3557    fn parse_session_output_exit1_provider_error_stays_agent_failed() {
3558        let ndjson =
3559            r#"{"type":"result","subtype":"error","is_error":true,"errors":["provider error"]}"#;
3560        let session = session::SessionOutput {
3561            exit_code: 1,
3562            raw_stdout: ndjson.to_string(),
3563            raw_stderr: String::new(),
3564            elapsed_secs: Some(12),
3565            sandbox_error: None,
3566        };
3567
3568        let runner = agents::get_runner("grok");
3569        let err = parse_session_output("4068", "grok", &*runner, &session)
3570            .expect_err("provider error should be an error");
3571        assert!(
3572            matches!(err, agents::AgentError::AgentFailed { .. }),
3573            "exit 1 provider error should stay AgentFailed, got {err:?}"
3574        );
3575    }
3576
3577    // ── terminal_reason:completed + exit 1 detection (issue #3087) ─────────────
3578
3579    /// kimi/minimax sometimes emit valid NDJSON with terminal_reason=completed
3580    /// but exit 1. In that case parse_success_output may fail (e.g. empty result
3581    /// text that synthesize can't parse), then parse_session_output would
3582    /// normally fall back to classify_error — grabbing a garbled JSON tail and
3583    /// producing a spurious Unknown/Auth error. The fix intercepts this and
3584    /// returns InvalidResponse (soft error, re-route) instead.
3585    #[test]
3586    fn parse_session_output_exit0_without_envelope_is_interrupted() {
3587        let session = session::SessionOutput {
3588            exit_code: 0,
3589            raw_stdout: "\u{4}\u{4}\u{4} truncated pane junk".to_string(),
3590            raw_stderr: String::new(),
3591            elapsed_secs: Some(180),
3592            sandbox_error: None,
3593        };
3594
3595        let runner = agents::get_runner("claude");
3596        let err = parse_session_output("3876", "claude", &*runner, &session)
3597            .expect_err("missing envelope should be an error");
3598        assert!(
3599            matches!(err, agents::AgentError::Interrupted { .. }),
3600            "exit 0 with no result envelope should be Interrupted, got {err:?}"
3601        );
3602    }
3603
3604    #[test]
3605    fn parse_session_output_exit1_with_terminal_reason_completed_returns_invalid_response() {
3606        // Empty result text causes synthesize_response_from_text to return None
3607        // (empty/whitespace-only input), so parse_success_output returns
3608        // InvalidResponse. With the fix, parse_session_output intercepts the
3609        // "terminal_reason":"completed" signal before classify_error and
3610        // returns InvalidResponse (re-route) instead of a garbled error.
3611        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":"","usage":{"input_tokens":1000,"output_tokens":100}}"#;
3612        let session = session::SessionOutput {
3613            exit_code: 1,
3614            raw_stdout: ndjson.to_string(),
3615            raw_stderr: String::new(),
3616            elapsed_secs: Some(30),
3617            sandbox_error: None,
3618        };
3619
3620        let runner = agents::get_runner("claude");
3621        let result = parse_session_output("149298", "claude", &*runner, &session);
3622
3623        let err = result.expect_err("should be an error, not Ok");
3624        assert!(
3625            matches!(err, agents::AgentError::InvalidResponse { .. }),
3626            "should be InvalidResponse, got {err:?}"
3627        );
3628    }
3629
3630    // ── GLM cost-telemetry exit-1 detection (issue #3094) ──────────────────────
3631
3632    /// GLM exits with code 1 but stdout is only cost telemetry JSON (contains
3633    /// "costUSD":). This must return InvalidResponse (soft re-route) instead of
3634    /// a hard Failed classification with spurious agent cooldown.
3635    #[test]
3636    fn parse_session_output_exit1_with_cost_telemetry_returns_invalid_response() {
3637        // GLM emits cost/usage JSON on clean exit; exit code 1 is a false failure.
3638        let telemetry = r#"{"type":"result","subtype":"success","is_error":false,"result":"","costUSD":0.042,"usage":{"input_tokens":5000,"output_tokens":800}}"#;
3639        let session = session::SessionOutput {
3640            exit_code: 1,
3641            raw_stdout: telemetry.to_string(),
3642            raw_stderr: String::new(),
3643            elapsed_secs: Some(120),
3644            sandbox_error: None,
3645        };
3646
3647        let runner = agents::get_runner("glm");
3648        let result = parse_session_output("149347", "glm", &*runner, &session);
3649
3650        let err = result.expect_err("should be an error, not Ok");
3651        assert!(
3652            matches!(err, agents::AgentError::InvalidResponse { .. }),
3653            "glm cost-telemetry exit-1 should return InvalidResponse, got {err:?}"
3654        );
3655    }
3656
3657    /// GLM with costUSD AND is_error:true — must propagate AgentFailed, not
3658    /// silently swallow the error via the cost-telemetry guard.
3659    #[test]
3660    fn parse_session_output_exit1_cost_telemetry_with_is_error_true_propagates_failure() {
3661        let telemetry = r#"{"type":"result","subtype":"error","is_error":true,"result":"tool failed","costUSD":0.01,"usage":{"input_tokens":100,"output_tokens":10}}"#;
3662        let session = session::SessionOutput {
3663            exit_code: 1,
3664            raw_stdout: telemetry.to_string(),
3665            raw_stderr: String::new(),
3666            elapsed_secs: Some(15),
3667            sandbox_error: None,
3668        };
3669
3670        let runner = agents::get_runner("glm");
3671        let result = parse_session_output("149348", "glm", &*runner, &session);
3672
3673        let err = result.expect_err("should be an error");
3674        assert!(
3675            matches!(err, agents::AgentError::AgentFailed { .. }),
3676            "is_error=true with costUSD must still return AgentFailed, got {err:?}"
3677        );
3678    }
3679
3680    /// terminal_reason=completed must NOT override actual error conditions.
3681    /// When is_error=true in the NDJSON envelope, AgentFailed must be returned
3682    /// even if terminal_reason=completed is also present. Only kimi/minimax
3683    /// agents use find_claude_result which extracts is_error, so use "kimi"
3684    /// as agent_name.
3685    #[test]
3686    fn parse_session_output_exit1_preserves_error_when_is_error_true() {
3687        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"terminal_reason":"completed","result":"failed"}"#;
3688        let session = session::SessionOutput {
3689            exit_code: 1,
3690            raw_stdout: ndjson.to_string(),
3691            raw_stderr: String::new(),
3692            elapsed_secs: Some(30),
3693            sandbox_error: None,
3694        };
3695
3696        let runner = agents::get_runner("kimi");
3697        let result = parse_session_output("149299", "kimi", &*runner, &session);
3698
3699        let err = result.expect_err("should be an error");
3700        assert!(
3701            matches!(err, agents::AgentError::AgentFailed { .. }),
3702            "is_error=true should return AgentFailed, got {err:?}"
3703        );
3704    }
3705
3706    /// RateLimit with terminal_reason=completed must be propagated as RateLimit,
3707    /// not downgraded to InvalidResponse. This covers claude session-limit errors
3708    /// that include "terminal_reason":"completed" in their NDJSON envelope — the
3709    /// terminal_reason guard was designed for kimi/minimax/glm false-positive
3710    /// exit codes but was intercepting classified errors like RateLimit.
3711    #[test]
3712    fn parse_session_output_exit1_with_rate_limit_and_terminal_reason_completed() {
3713        let ndjson = r#"{"type":"result","is_error":true,"terminal_reason":"completed","result":"You've hit your session limit · resets 2:20pm (America/Sao_Paulo)","usage":{"input_tokens":0,"output_tokens":0}}"#;
3714        let session = session::SessionOutput {
3715            exit_code: 1,
3716            raw_stdout: ndjson.to_string(),
3717            raw_stderr: String::new(),
3718            elapsed_secs: Some(120),
3719            sandbox_error: None,
3720        };
3721
3722        let runner = agents::get_runner("claude");
3723        let result = parse_session_output("152895", "claude", &*runner, &session);
3724
3725        let err = result.expect_err("rate-limit should be an error");
3726        assert!(
3727            matches!(err, agents::AgentError::RateLimit { .. }),
3728            "session limit should return RateLimit, not downgraded, got {err:?}"
3729        );
3730    }
3731}