Skip to main content

orch/engine/runner/
mod.rs

1//! Task runner — executes tasks using AI agents.
2//!
3//! Replaces `run_task.sh` with pure Rust. The runner:
4//! 1. Sets up a git worktree for isolation
5//! 2. Builds context (project instructions, skills, comments)
6//! 3. Renders prompts and spawns the agent in tmux
7//! 4. Waits for completion, collects and classifies the response
8//! 5. Handles errors (reroute on limits, fallback agents)
9//! 6. Auto-commits, pushes, and creates PRs
10//! 7. Updates status labels and posts result comments
11//!
12//! Submodules handle the heavy lifting:
13//! - [`task_init`] — guard checks, worktree setup, and invocation building
14//! - [`session`] — tmux session lifecycle and output collection
15//! - [`response_handler`] — success path: commit, push, PR, token storage
16//! - [`fallback`] — error classification and recovery strategies
17
18pub mod agent;
19pub mod agents;
20pub mod commit;
21pub mod context;
22pub mod cow;
23pub mod diff;
24pub mod direct;
25pub mod fallback;
26pub mod git_ops;
27pub mod response;
28pub mod response_handler;
29pub mod sandbox;
30pub mod session;
31pub mod task_init;
32pub mod worktree;
33
34use crate::backends::{ExternalBackend, ExternalTask, Status};
35use crate::config;
36use crate::engine::router::RouteResult;
37use crate::engine::tasks::is_internal_id;
38use crate::store;
39use crate::store::InsertTaskMetric;
40use crate::tmux::TmuxManager;
41use anyhow::Context;
42use chrono::Utc;
43pub use response::WeightSignal;
44use std::path::PathBuf;
45use std::sync::Arc;
46
47/// Fully materialized task-run audit data.
48#[derive(Debug, Clone)]
49pub struct RunAudit {
50    pub stdout: String,
51    pub stderr: String,
52    pub parsed_response: String,
53    pub outcome: String,
54    pub error: String,
55    pub input_tokens: u64,
56    pub output_tokens: u64,
57    pub total_cost_usd: f64,
58    pub duration_secs: f64,
59}
60
61/// Result returned by `run()` so callers can persist the audit trail.
62#[derive(Debug, Clone)]
63pub struct RunExecution {
64    pub status: String,
65    pub exit_code: Option<i32>,
66    pub audit: RunAudit,
67}
68
69fn serialize_parsed_response(
70    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
71    raw_stdout: &str,
72) -> String {
73    match parse_result {
74        Ok(parsed) => serde_json::json!({
75            "response": parsed.response,
76            "input_tokens": parsed.input_tokens,
77            "output_tokens": parsed.output_tokens,
78            "duration_ms": parsed.duration_ms,
79        })
80        .to_string(),
81        Err(agents::AgentError::InvalidResponse { raw }) => {
82            serde_json::json!({"raw": raw}).to_string()
83        }
84        Err(_) => serde_json::json!({"raw": raw_stdout}).to_string(),
85    }
86}
87
88fn classify_run_outcome(
89    status: &str,
90    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
91    push_failed: bool,
92) -> &'static str {
93    if push_failed {
94        return "push_failed";
95    }
96    match parse_result {
97        Err(agents::AgentError::Timeout { .. }) => "timeout",
98        Err(agents::AgentError::Interrupted { .. }) => "interrupted",
99        // Generic rate-limit patterns (e.g. "usage limit") also match billing-cycle
100        // exhaustion messages, so consult the credit-exhaustion detector first —
101        // otherwise Kimi-style "usage limit for this billing cycle" errors are
102        // stored as a plain "rate_limit" outcome (issue #3529).
103        Err(agents::AgentError::RateLimit { message }) => {
104            crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
105                .unwrap_or("rate_limit")
106        }
107        Err(agents::AgentError::Auth { message }) => {
108            crate::engine::cooldown::detect_credit_exhaustion_outcome(message).unwrap_or("failed")
109        }
110        Err(agents::AgentError::InvalidResponse { .. }) => "parse_error",
111        Err(_) => "failed",
112        Ok(_)
113            if matches!(
114                status,
115                "done" | "completed" | "in_progress" | "in_review" | "needs_review" | "routed"
116            ) =>
117        {
118            "success"
119        }
120        Ok(_) if status == "blocked" => "blocked",
121        // Non-canonical status that clearly reads as completion — success, not
122        // "unrecognized status" (#3627).
123        Ok(_) if crate::status_heuristics::status_looks_like_descriptive_completion(status) => {
124            "success"
125        }
126        Ok(_) => "failed",
127    }
128}
129
130fn classify_run_error_type(last_error: &str) -> &'static str {
131    if last_error.is_empty() {
132        // No error: agent completed successfully and created a PR waiting for review
133        "success"
134    } else if last_error.contains("timeout") {
135        "timeout"
136    } else if last_error.contains("interrupted") {
137        "interrupted"
138    } else if last_error.contains("billing cycle") || last_error.contains("monthly quota") {
139        "billing_cycle_exhausted"
140    } else if last_error.contains("rate limit") || last_error.contains("usage limit") {
141        "rate_limit"
142    } else if last_error.contains("auth") {
143        "auth_error"
144    } else if last_error.contains("push failed") {
145        "push_failed"
146    } else if last_error.contains("No commits between") || last_error.contains("no commits") {
147        "no_commits"
148    } else if last_error.contains("create PR failed")
149        || last_error.contains("failed to create pull request")
150        || last_error.contains("pull request creation failed")
151    {
152        "pr_failed"
153    } else if last_error.contains("invalid response") || last_error.contains("parse error") {
154        "parse_error"
155    } else if last_error.contains("exceeded max attempts") {
156        "max_attempts"
157    } else {
158        let lower = last_error.to_ascii_lowercase();
159        if lower.contains("build failed")
160            || lower.contains("tests failed")
161            || lower.contains("test failed")
162            || lower.contains("test suite failed")
163            || lower.contains("lint failed")
164        {
165            "ci_failure"
166        } else {
167            "failed"
168        }
169    }
170}
171
172fn extract_run_tokens(
173    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
174) -> (u64, u64) {
175    match parse_result {
176        Ok(parsed) => (
177            parsed
178                .input_tokens
179                .or(parsed.response.input_tokens)
180                .unwrap_or(0),
181            parsed
182                .output_tokens
183                .or(parsed.response.output_tokens)
184                .unwrap_or(0),
185        ),
186        Err(_) => (0, 0),
187    }
188}
189
190fn run_duration_secs(started_at: &chrono::DateTime<Utc>) -> f64 {
191    (Utc::now() - *started_at).num_milliseconds() as f64 / 1000.0
192}
193
194struct RunAuditInput<'a> {
195    task_id: &'a str,
196    status: &'a str,
197    parse_result: &'a Result<agents::ParsedResponse, agents::AgentError>,
198    raw_stdout: &'a str,
199    raw_stderr: &'a str,
200    started_at: &'a chrono::DateTime<Utc>,
201    error_override: Option<String>,
202    elapsed_secs: Option<u64>,
203    push_failed: bool,
204}
205
206fn parse_success_output(
207    task_id: &str,
208    agent_name: &str,
209    agent_runner: &dyn agents::AgentRunner,
210    raw_stdout: &str,
211) -> Result<agents::ParsedResponse, agents::AgentError> {
212    // Quick NDJSON terminal_reason:completed shortcut: when the agent
213    // emitted a session envelope that explicitly signals completion we
214    // prefer to interpret that as a successful run (or at least a
215    // soft/empty success) rather than immediately classifying an error
216    // from the trailing process output.  Do not override explicit
217    // in-envelope `is_error: true` signals.
218    if raw_stdout.contains("\"terminal_reason\":\"completed\"")
219        && !raw_stdout.contains("\"is_error\":true")
220    {
221        // Try to recover a structured response from assistant messages first
222        let assistant_text = agents::collect_assistant_messages_text(agent_name, raw_stdout);
223        if !assistant_text.is_empty() {
224            if let Ok(response) = crate::parser::parse(&assistant_text) {
225                tracing::debug!(task_id, agent = agent_name, "found AgentResponse JSON in earlier assistant message (terminal_reason:completed)");
226                return Ok(agents::ParsedResponse {
227                    response,
228                    input_tokens: None,
229                    output_tokens: None,
230                    duration_ms: None,
231                });
232            }
233            if let Some(response) = agents::synthesize_response_from_text(&assistant_text) {
234                tracing::warn!(
235                    task_id,
236                    agent = agent_name,
237                    "synthesizing response from assistant message due to terminal_reason:completed"
238                );
239                return Ok(agents::ParsedResponse {
240                    response,
241                    input_tokens: None,
242                    output_tokens: None,
243                    duration_ms: None,
244                });
245            }
246        }
247        // If we couldn't recover structured data from earlier assistant
248        // messages, fall through and let the regular NDJSON envelope parsing
249        // / agent-specific parsers handle the terminal_reason case. This
250        // preserves the existing behavior where an empty `result` field
251        // combined with exit != 0 becomes an InvalidResponse that is then
252        // interpreted by parse_session_output (see tests).
253    }
254
255    // 1. Try to use the unified agent result extractor first
256    if let Some(agent_result) = agents::find_agent_result(agent_name, raw_stdout) {
257        // Treat is_error as authoritative for the success path
258        if agent_result.is_error {
259            // Classify the error from the agent's result text rather than
260            // blindly returning AgentFailed — this preserves rate-limit, auth,
261            // and other actionable error types for proper cooldown/reroute.
262            // Without this, e.g. a claude session-limit error with is_error=true
263            // would be downgraded to AgentFailed, losing the RateLimit signal
264            // that the fallback chain needs for UsageLimit recovery (issue #3292).
265            if let Some(e) = agents::patterns::detect_rate_limit(&agent_result.result_text) {
266                return Err(e);
267            }
268            if let Some(e) = agents::patterns::detect_auth_error(&agent_result.result_text) {
269                return Err(e);
270            }
271            if let Some(e) = agents::patterns::detect_model_unavailable(&agent_result.result_text) {
272                return Err(e);
273            }
274            return Err(agents::AgentError::AgentFailed {
275                message: agent_result.result_text,
276            });
277        }
278
279        // Try to parse the inner result text as standard JSON
280        match crate::parser::parse(&agent_result.result_text) {
281            Ok(response) => {
282                return Ok(agents::ParsedResponse {
283                    response,
284                    input_tokens: agent_result.input_tokens,
285                    output_tokens: agent_result.output_tokens,
286                    duration_ms: agent_result.duration_ms,
287                });
288            }
289            Err(_) => {
290                // Strategy 2: scan all earlier assistant messages in the NDJSON
291                // stream for valid AgentResponse JSON. Agents often emit the
292                // JSON in an intermediate turn and then summarise in prose as
293                // their final output, so the `type:result` text may be pure
294                // narrative while a prior assistant turn contains the JSON.
295                let assistant_text =
296                    agents::collect_assistant_messages_text(agent_name, raw_stdout);
297                if !assistant_text.is_empty() {
298                    if let Ok(response) = crate::parser::parse(&assistant_text) {
299                        tracing::debug!(
300                            task_id,
301                            agent = agent_name,
302                            "found AgentResponse JSON in earlier assistant message"
303                        );
304                        return Ok(agents::ParsedResponse {
305                            response,
306                            input_tokens: agent_result.input_tokens,
307                            output_tokens: agent_result.output_tokens,
308                            duration_ms: agent_result.duration_ms,
309                        });
310                    }
311                }
312
313                // Only fall back to text synthesis when parsing fails
314                if let Some(response) =
315                    agents::synthesize_response_from_text(&agent_result.result_text)
316                {
317                    tracing::warn!(
318                        task_id,
319                        agent = agent_name,
320                        "parse failed on agent result, synthesizing response from plain text"
321                    );
322                    return Ok(agents::ParsedResponse {
323                        response,
324                        input_tokens: agent_result.input_tokens,
325                        output_tokens: agent_result.output_tokens,
326                        duration_ms: agent_result.duration_ms,
327                    });
328                }
329
330                // NDJSON envelope fallback: the envelope explicitly reports
331                // success (is_error=false) but the result content is domain
332                // JSON or plain prose that doesn't match AgentResponse schema
333                // and doesn't contain completion keywords.  Rather than
334                // returning InvalidResponse (which causes the task to re-run),
335                // synthesize a minimal "done" response from the result text.
336                //
337                // Guard against kimi/minimax edge case: when terminal_reason=completed
338                // but result text is empty, synthesize returns None, parse_success_output
339                // returns InvalidResponse, and parse_session_output can intercept it
340                // before classify_error. Requiring non-empty result here means the
341                // envelope fallback only fires when we have actual text to report.
342                if !agent_result.is_error
343                    && result_envelope_completed(raw_stdout)
344                    && !agent_result.result_text.is_empty()
345                {
346                    let summary_end =
347                        agents::truncate_at_char_boundary(&agent_result.result_text, 500);
348                    let summary = agent_result.result_text[..summary_end].to_string();
349                    tracing::warn!(
350                        task_id,
351                        agent = agent_name,
352                        "NDJSON envelope indicates success but result lacks AgentResponse \
353                         schema — synthesizing done response"
354                    );
355                    return Ok(agents::ParsedResponse {
356                        response: crate::parser::AgentResponse {
357                            status: "done".to_string(),
358                            summary,
359                            ..Default::default()
360                        },
361                        input_tokens: agent_result.input_tokens,
362                        output_tokens: agent_result.output_tokens,
363                        duration_ms: agent_result.duration_ms,
364                    });
365                }
366            }
367        }
368    }
369
370    // 2. Fall back to generic agent-specific parser if extraction failed
371    match agent_runner.parse_response(raw_stdout) {
372        Ok(parsed) => Ok(parsed),
373        Err(agents::AgentError::InvalidResponse { raw }) => {
374            if let Some(response) = agents::synthesize_response_from_text(&raw) {
375                tracing::warn!(
376                    task_id,
377                    agent = agent_name,
378                    "parse failed, synthesizing response from plain text"
379                );
380                // Try to extract token metadata from the original raw NDJSON
381                // output (not the truncated/extracted text from the error), so
382                // cost tracking isn't lost for agents that return free-form text.
383                let agent_result = agents::find_agent_result(agent_name, raw_stdout);
384                let input_tokens = agent_result.as_ref().and_then(|r| r.input_tokens);
385                let output_tokens = agent_result.as_ref().and_then(|r| r.output_tokens);
386                let duration_ms = agent_result.as_ref().and_then(|r| r.duration_ms);
387                Ok(agents::ParsedResponse {
388                    response,
389                    input_tokens,
390                    output_tokens,
391                    duration_ms,
392                })
393            } else {
394                Err(agents::AgentError::InvalidResponse { raw })
395            }
396        }
397        Err(err) => Err(err),
398    }
399}
400
401/// Whether the last `type: result` line reports `subtype: success` or
402/// `terminal_reason: completed`. Grok omits `terminal_reason`.
403fn result_envelope_line(raw_stdout: &str) -> Option<serde_json::Value> {
404    raw_stdout
405        .lines()
406        .rev()
407        .filter_map(|l| serde_json::from_str::<serde_json::Value>(l.trim()).ok())
408        .find(|v| v.get("type").and_then(|t| t.as_str()) == Some("result"))
409}
410
411fn has_result_envelope(raw_stdout: &str) -> bool {
412    result_envelope_line(raw_stdout).is_some()
413}
414
415fn result_envelope_completed(raw_stdout: &str) -> bool {
416    result_envelope_line(raw_stdout).is_some_and(|v| {
417        v.get("subtype").and_then(|t| t.as_str()) == Some("success")
418            || v.get("terminal_reason").and_then(|t| t.as_str()) == Some("completed")
419    })
420}
421
422// A timeout kill has no result envelope. Salvage only the wrap-up shape: a successful
423// `git commit`, then only shell calls that all returned ok. Returns the last assistant text
424fn salvage_wrapup(raw: &str) -> Option<String> {
425    use serde_json::Value;
426    // (tool name, command, finished ok) in call order
427    let mut calls: Vec<(String, String, Option<bool>)> = Vec::new();
428    let mut last_text = String::new();
429    for v in raw
430        .lines()
431        .filter_map(|l| serde_json::from_str::<Value>(l.trim()).ok())
432    {
433        let blocks = v.pointer("/message/content").and_then(|c| c.as_array());
434        for b in blocks.into_iter().flatten() {
435            match b.get("type").and_then(|t| t.as_str()) {
436                Some("tool_use") => calls.push((
437                    b["name"].as_str().unwrap_or("").to_string(),
438                    b.pointer("/input/command")
439                        .and_then(|c| c.as_str())
440                        .unwrap_or("")
441                        .to_string(),
442                    None,
443                )),
444                Some("text") if v["type"] == "assistant" => {
445                    last_text = b["text"].as_str().unwrap_or("").to_string();
446                }
447                Some("tool_result") => {
448                    let ok = b.get("is_error").and_then(|e| e.as_bool()) != Some(true);
449                    // results arrive in call order, fill the first unanswered call
450                    if let Some(c) = calls.iter_mut().find(|c| c.2.is_none()) {
451                        c.2 = Some(ok);
452                    }
453                }
454                _ => {}
455            }
456        }
457    }
458    let commit = calls
459        .iter()
460        .rposition(|(_, cmd, ok)| cmd.contains("git commit") && *ok == Some(true))?;
461    let tail_ok = calls[commit..].iter().all(|(name, _, ok)| {
462        matches!(name.as_str(), "Bash" | "run_terminal_cmd") && *ok == Some(true)
463    });
464    let lower = last_text.to_lowercase();
465    if !tail_ok || lower.contains("still running") || lower.contains("in progress") {
466        return None;
467    }
468    Some(last_text.chars().take(500).collect())
469}
470
471/**
472 * Tries `parse_success_output` first when stdout is non-empty. If that succeeds,
473 * returns success regardless of exit code. If it fails and `exit_code != 0`,
474 * checks for `"terminal_reason":"completed"` or cost-telemetry-only stdout
475 * (`"costUSD":`) before falling back to `classify_error`, so valid sessions that
476 * exit 1 produce `InvalidResponse` (soft error, re-route) instead of a garbled
477 * `Unknown`/`Auth` error (hard failure with spurious cooldown)
478 **/
479pub fn parse_session_output(
480    task_id: &str,
481    agent_name: &str,
482    agent_runner: &dyn agents::AgentRunner,
483    session_output: &session::SessionOutput,
484) -> Result<agents::ParsedResponse, agents::AgentError> {
485    // Some agents may emit NDJSON/telemetry to stderr instead of stdout. Combine
486    // both streams so parse_success_output can inspect either place for the
487    // terminal_reason/cost telemetry envelope. Only skip parsing when both are
488    // empty so we still attempt recovery when stderr contains the envelope.
489    let combined_output = format!("{}{}", session_output.raw_stdout, session_output.raw_stderr);
490    if !(session_output.raw_stdout.is_empty() && session_output.raw_stderr.is_empty()) {
491        match parse_success_output(task_id, agent_name, agent_runner, &combined_output) {
492            Ok(parsed) => return Ok(parsed),
493            Err(err) => {
494                if session_output.exit_code == 124 {
495                    if let Some(summary) = salvage_wrapup(&combined_output) {
496                        tracing::warn!(
497                            task_id,
498                            agent = agent_name,
499                            "timed out after commit during wrap-up, salvaging as done"
500                        );
501                        return Ok(agents::ParsedResponse {
502                            response: crate::parser::AgentResponse {
503                                status: "done".to_string(),
504                                summary,
505                                ..Default::default()
506                            },
507                            input_tokens: None,
508                            output_tokens: None,
509                            duration_ms: None,
510                        });
511                    }
512                }
513                // If parse_success_output already returned a classified error
514                // (RateLimit, AgentFailed, Timeout, Auth, etc.), propagate it
515                // directly — only InvalidResponse (unparseable output) should
516                // be intercepted by the terminal_reason:completed / cost-telemetry
517                // guard below. Without this, e.g. a claude session-limit error
518                // with terminal_reason=completed would be downgraded from RateLimit
519                // to InvalidResponse → Failed, losing the correct cooldown/reroute.
520                if !matches!(err, agents::AgentError::InvalidResponse { .. }) {
521                    return Err(err);
522                }
523                if session_output.exit_code != 0 {
524                    // kimi/minimax/glm sometimes emit valid NDJSON with
525                    // "terminal_reason":"completed" or cost-only telemetry
526                    // ("costUSD":) but exit 1 due to a post-session hook or
527                    // minor CLI issue. Scanning the tail would just grab JSON
528                    // metadata, producing a garbled error. Detect either
529                    // completion signal and treat as a soft parse error so the
530                    // task re-routes instead of hitting a false cooldown from
531                    // Unknown or Auth error variants. Use the combined output
532                    // (stdout+stderr) so envelopes written to stderr are
533                    // observed.
534                    let combined = &combined_output;
535                    let is_completed_signal =
536                        combined.contains("\"terminal_reason\":\"completed\"");
537                    // GLM emits a cost-telemetry JSON object (containing
538                    // "costUSD":) as its final stdout/stderr line on clean
539                    // exit even when the process exits with code 1. If the
540                    // only content is such telemetry (no is_error=true in the
541                    // envelope), treat it the same as terminal_reason:completed.
542                    let is_cost_telemetry_only = combined.contains("\"costUSD\":")
543                        && !combined.contains("\"is_error\":true");
544                    if is_completed_signal || is_cost_telemetry_only {
545                        let raw_tail = agents::patterns::safe_tail(combined, 300);
546                        return Err(agents::AgentError::InvalidResponse {
547                            raw: raw_tail.to_string(),
548                        });
549                    }
550                    return Err(agent_runner.classify_error_with_elapsed(
551                        session_output.exit_code,
552                        &session_output.raw_stdout,
553                        &session_output.raw_stderr,
554                        session_output.elapsed_secs,
555                    ));
556                }
557                if !has_result_envelope(&combined_output) {
558                    return Err(agents::AgentError::Interrupted {
559                        message: "exited without a result envelope".to_string(),
560                    });
561                }
562                return Err(err);
563            }
564        }
565    }
566
567    if session_output.exit_code != 0 {
568        return Err(agent_runner.classify_error_with_elapsed(
569            session_output.exit_code,
570            &session_output.raw_stdout,
571            &session_output.raw_stderr,
572            session_output.elapsed_secs,
573        ));
574    }
575
576    Err(agents::AgentError::Unknown {
577        exit_code: session_output.exit_code,
578        message: "empty-output-exit0: opencode returned exit 0 with empty stdout".to_string(),
579    })
580}
581
582/// Task runner configuration.
583pub struct TaskRunner {
584    /// Repository slug (owner/repo)
585    repo: String,
586    /// Path to the orch home directory
587    orch_home: PathBuf,
588    /// Unified task store for run audit trail
589    store: Option<Arc<crate::store::TaskStore>>,
590}
591
592impl TaskRunner {
593    pub fn new(repo: String) -> Self {
594        let orch_home = crate::home::orch_home().unwrap_or_else(|e| {
595            tracing::warn!(error = %e, "orch_home() failed — falling back to /tmp/.orch");
596            PathBuf::from("/tmp").join(".orch")
597        });
598
599        Self {
600            repo,
601            orch_home,
602            store: None,
603        }
604    }
605
606    /// Set the unified task store for run audit trail.
607    pub fn with_store(mut self, store: Arc<crate::store::TaskStore>) -> Self {
608        self.store = Some(store);
609        self
610    }
611
612    /// Read a field from the task store.
613    async fn get_field(&self, task_id: &str, field: &str) -> Option<String> {
614        let task = store::opt_store_get_task(&self.store, &self.repo, task_id).await?;
615        match field {
616            "route_reason" => Some(task.route_reason),
617            "worktree" => Some(task.worktree),
618            "attempts" => Some(task.attempts.to_string()),
619            "delegations" => serde_json::to_string(&task.delegations).ok(),
620            "summary" => Some(task.summary),
621            "last_error" => Some(task.last_error),
622            "agent" => task.agent,
623            _ => None,
624        }
625    }
626
627    async fn build_run_audit(&self, input: RunAuditInput<'_>) -> RunAudit {
628        // Filter empty strings so that a blank last_error does not shadow the
629        // actual parse_result error.  Without this, `Option::or(Some(""))` would
630        // return `Some("")` and prevent the `unwrap_or_else` fallback from ever
631        // running, causing task_runs.error to be written as "" even when the
632        // agent returned a classifiable error (fixes issue #2527).
633        let last_error = self
634            .get_field(input.task_id, "last_error")
635            .await
636            .filter(|s| !s.is_empty());
637        let outcome =
638            classify_run_outcome(input.status, input.parse_result, input.push_failed).to_string();
639
640        let error = if outcome == "success" {
641            String::new()
642        } else {
643            // Prefer this run's parsed error over a last_error left by a previous
644            // agent's failover. last_error stays the Err/EarlyReturn fallback (#3851)
645            let error = input
646                .error_override
647                .as_deref()
648                .filter(|s| !s.is_empty())
649                .map(str::to_string)
650                .or_else(|| match input.parse_result {
651                    Ok(parsed) => parsed
652                        .response
653                        .error
654                        .as_deref()
655                        .filter(|s| !s.is_empty())
656                        .map(str::to_string),
657                    Err(_) => None,
658                })
659                .or(last_error)
660                .unwrap_or_else(|| match input.parse_result {
661                    Ok(_) => String::new(),
662                    Err(err) => err.to_string(),
663                });
664
665            if error.is_empty() {
666                if input.status == "blocked" {
667                    match input.parse_result {
668                        Ok(parsed) if !parsed.response.summary.is_empty() => {
669                            parsed.response.summary.clone()
670                        }
671                        _ => "task blocked without explicit error".to_string(),
672                    }
673                } else if matches!(
674                    input.status,
675                    "done"
676                        | "completed"
677                        | "in_progress"
678                        | "in_review"
679                        | "needs_review"
680                        | "routed"
681                        | "new"
682                ) {
683                    // Canonical non-success statuses have no error message (e.g. "routed").
684                    String::new()
685                } else {
686                    // Non-canonical status with no error message — surface the status
687                    // explicitly so operators can identify which statuses need to be
688                    // added to the normalization map in parser.rs.
689                    format!("unrecognized status: {}", input.status)
690                }
691            } else {
692                error
693            }
694        };
695
696        let total_cost_usd = match &self.store {
697            Some(_) => {
698                store::get_cost_estimate(&self.store, &self.repo, input.task_id)
699                    .await
700                    .total_cost_usd
701            }
702            None => 0.0,
703        };
704
705        let (input_tokens, output_tokens) = extract_run_tokens(input.parse_result);
706
707        let duration_secs = input
708            .elapsed_secs
709            .map(|s| s as f64)
710            .unwrap_or_else(|| run_duration_secs(input.started_at));
711
712        RunAudit {
713            stdout: input.raw_stdout.to_string(),
714            stderr: input.raw_stderr.to_string(),
715            parsed_response: serialize_parsed_response(input.parse_result, input.raw_stdout),
716            outcome,
717            error,
718            input_tokens,
719            output_tokens,
720            total_cost_usd,
721            duration_secs,
722        }
723    }
724
725    /// The blocked outcome for a sandbox that cannot start. No agent ran, so none is cooled down
726    async fn sandbox_blocked(&self, task_id: &str, reason: String) -> RunExecution {
727        tracing::error!(task_id, error = %reason, "blocking task");
728        if let Err(e) = store::store_set_result(
729            &self.store,
730            &self.repo,
731            task_id,
732            &[
733                ("last_error", serde_json::json!(reason)),
734                ("block_reason", serde_json::json!(reason)),
735            ],
736        )
737        .await
738        {
739            tracing::warn!(task_id, err = %e, "failed to write the sandbox block_reason");
740        }
741        RunExecution {
742            status: "blocked".to_string(),
743            exit_code: None,
744            audit: RunAudit {
745                stdout: String::new(),
746                stderr: String::new(),
747                parsed_response: String::new(),
748                outcome: "failed".to_string(),
749                error: reason,
750                input_tokens: 0,
751                output_tokens: 0,
752                total_cost_usd: 0.0,
753                duration_secs: 0.0,
754            },
755        }
756    }
757
758    /// Run a task through the full execution pipeline.
759    ///
760    /// Returns `Ok(None)` if the runner guard skipped the task (caller should not
761    /// post stale results). Returns `Ok(Some(status))` with the final task status.
762    pub async fn run(
763        &self,
764        task_id: &str,
765        agent: Option<&str>,
766        model: Option<&str>,
767        backend: Option<&dyn ExternalBackend>,
768        started_at: &chrono::DateTime<Utc>,
769    ) -> anyhow::Result<Option<RunExecution>> {
770        tracing::info!(
771            task_id,
772            agent = agent.unwrap_or("default"),
773            model = model.unwrap_or("default"),
774            "starting task execution"
775        );
776
777        // Check task guards; returns outcome indicating whether to proceed.
778        let attempts = match task_init::check_guards(task_id, &self.repo, &self.store).await {
779            Ok(task_init::GuardOutcome::Proceed(a)) => a,
780            Ok(task_init::GuardOutcome::Skip) => {
781                return Ok(None);
782            }
783            Ok(task_init::GuardOutcome::MaxAttempts) => {
784                // The caller marks the task blocked from the returned status, a task that ran out
785                // of attempts did not finish, so it must never reach done
786                if backend.is_some() {
787                    // If this was label-forced to a specific agent, remove the agent label
788                    // so that /retry can route to a different agent instead of looping.
789                    let route_reason = self
790                        .get_field(task_id, "route_reason")
791                        .await
792                        .unwrap_or_default();
793                    if route_reason.starts_with("label agent:") {
794                        let agent_label = route_reason.trim_start_matches("label ");
795                        let gh = crate::github::http::GhHttp::new()?;
796                        if let Err(e) = gh.remove_label(&self.repo, task_id, agent_label).await {
797                            tracing::warn!(task_id, label = agent_label, err = %e, "failed to remove agent label after max attempts");
798                        } else {
799                            tracing::info!(task_id, label = agent_label, "removed forced agent label after max attempts — /retry will use free routing");
800                        }
801                    }
802                }
803                return Ok(Some(RunExecution {
804                    status: "blocked".to_string(),
805                    exit_code: None,
806                    audit: RunAudit {
807                        stdout: String::new(),
808                        stderr: String::new(),
809                        parsed_response: String::new(),
810                        outcome: "failed".to_string(),
811                        error: "max attempts reached".to_string(),
812                        input_tokens: 0,
813                        output_tokens: 0,
814                        total_cost_usd: 0.0,
815                        duration_secs: 0.0,
816                    },
817                }));
818            }
819            Err(e) => return Err(e),
820        };
821
822        // Resolve project directory
823        let project_dir = self.resolve_project_dir().await?;
824
825        // Prepare task: worktree, context, prompts, and agent invocation
826        let init = task_init::prepare_task(
827            task_id,
828            agent,
829            model,
830            backend,
831            &self.repo,
832            &project_dir,
833            attempts,
834            &self.store,
835        )
836        .await?;
837
838        // A sandbox that cannot start blocks the task, which never runs without it. It is the
839        // operator's setup that fails, so no agent or model is cooled down
840        if init.invocation.sandbox {
841            if let Err(e) = sandbox::argv(
842                &init.invocation,
843                &std::env::temp_dir().join("orch-sandbox-preflight"),
844            )
845            .await
846            {
847                return Ok(Some(self.sandbox_blocked(task_id, e.to_string()).await));
848            }
849        }
850
851        // Run agent session: spawn in tmux, wait for completion, collect output
852        let (tmux, tmux_session, session_output) = session::run_agent_session(
853            task_id,
854            &init.invocation,
855            &init.attempt_dir,
856            &self.orch_home,
857        )
858        .await;
859
860        // The pre-flight policy passed but the one built at spawn did not: block the same way
861        if let Some(reason) = session_output.sandbox_error.clone() {
862            return Ok(Some(self.sandbox_blocked(task_id, reason).await));
863        }
864
865        // Touch updated_at immediately after session exit so the stuck-task recovery
866        // timer resets before response_handler post-processing (git push, PR creation,
867        // status update) begins.  Without this, a session that ran longer than
868        // no_session_stuck_timeout (default 10 min) would be re-dispatched by the tick
869        // while the runner is still writing results.
870        store::store_touch_updated_at(&self.store, &self.repo, task_id).await;
871
872        // If silence detection (tick phase1b) already rerouted this task while the
873        // session was running, skip all fallback/review processing. Without this check the
874        // runner would call `fallback::handle_error` on the empty output, exhaust fallback
875        // agents, and mark the task `needs_review` — overwriting the rerouted status and
876        // triggering a spurious review cycle.
877        // Silence detection now sets Routed (with fallback agent) or NeedsReview (no fallback).
878        // Check for both, plus New for backward compatibility.
879        if let Some(stored) = store::opt_store_get_task(&self.store, &self.repo, task_id).await {
880            let silence_reset = matches!(
881                stored.status,
882                crate::store::TaskStatus::New
883                    | crate::store::TaskStatus::Routed
884                    | crate::store::TaskStatus::NeedsReview
885            );
886            if silence_reset {
887                let status_str = stored.status.as_str();
888                tracing::info!(
889                    task_id,
890                    status = status_str,
891                    "task already reset by silence detection — skipping fallback processing"
892                );
893                session::cleanup_session(task_id, &tmux, &tmux_session).await;
894                let silence_err = agents::patterns::classify_from_text(0, "silence detected");
895                let silence_parse: Result<agents::ParsedResponse, agents::AgentError> =
896                    Err(silence_err);
897                let audit = self
898                    .build_run_audit(RunAuditInput {
899                        task_id,
900                        status: status_str,
901                        parse_result: &silence_parse,
902                        raw_stdout: &session_output.raw_stdout,
903                        raw_stderr: &session_output.raw_stderr,
904                        started_at,
905                        error_override: Some(format!("silence detection set task to {status_str}")),
906                        elapsed_secs: session_output.elapsed_secs,
907                        push_failed: false,
908                    })
909                    .await;
910                return Ok(Some(RunExecution {
911                    status: status_str.to_string(),
912                    exit_code: Some(session_output.exit_code),
913                    audit,
914                }));
915            }
916        }
917
918        // Log raw output for debugging agent failures
919        let stdout_len = session_output.raw_stdout.len();
920        let stderr_len = session_output.raw_stderr.len();
921        let stdout_tail: String = session_output
922            .raw_stdout
923            .chars()
924            .rev()
925            .take(500)
926            .collect::<String>()
927            .chars()
928            .rev()
929            .collect();
930        let stderr_tail: String = session_output
931            .raw_stderr
932            .chars()
933            .rev()
934            .take(500)
935            .collect::<String>()
936            .chars()
937            .rev()
938            .collect();
939        tracing::info!(
940            task_id,
941            exit_code = session_output.exit_code,
942            stdout_len,
943            stderr_len,
944            stdout_tail = %stdout_tail,
945            stderr_tail = %stderr_tail,
946            "agent raw output"
947        );
948
949        // Use agent-specific parsing where possible. Previously we only attempted
950        // parse_success_output when exit_code == 0 which caused runs that emitted
951        // valid NDJSON but exited non-zero (kimi / minimax) to be classed as
952        // failures. Try parsing success output first when stdout is present; if
953        // parsing succeeds treat as success regardless of exit code. Only fall
954        // back to error classification when parsing fails and the process exit
955        // code indicates an error.
956        let agent_runner = agents::get_runner(&init.agent_name);
957        let parse_result =
958            parse_session_output(task_id, &init.agent_name, &*agent_runner, &session_output);
959
960        // Write structured result.json for deterministic testing and debugging
961        response_handler::write_result_json(
962            &init.attempt_dir,
963            task_id,
964            &init.agent_name,
965            init.model_name.as_deref(),
966            session_output.exit_code,
967            init.new_attempts,
968            &parse_result,
969            &session_output.raw_stdout,
970            &session_output.raw_stderr,
971        )
972        .await;
973
974        // Handle outcome: success or error recovery
975        // `fallback_error` carries the attributed error message from `handle_error()` when
976        // the Continue path is taken.  It is passed as `error_override` to `build_run_audit`
977        // so that stale `last_error` values from a previous agent's run are never written
978        // into the current run's audit record (fixes misattribution bug: task_run shows
979        // agent=kimi but error="minimax timed out...").
980        let (final_status, push_failed, fallback_error) = match parse_result {
981            Ok(ref parsed) => {
982                let (status, push_failed) = response_handler::handle_success(
983                    task_id,
984                    parsed.clone(),
985                    &init.wt,
986                    &init.task_title,
987                    &init.agent_name,
988                    init.model_name.as_deref(),
989                    init.new_attempts,
990                    &self.repo,
991                    &self.store,
992                    &session_output.raw_stdout,
993                )
994                .await?;
995                (status, push_failed, None)
996            }
997            Err(ref agent_err) => {
998                match fallback::handle_error(
999                    task_id,
1000                    agent_err,
1001                    &init.agent_name,
1002                    &*agent_runner,
1003                    init.model_name.as_deref(),
1004                    init.complexity.as_deref(),
1005                    init.new_attempts,
1006                    &self.store,
1007                    &self.repo,
1008                )
1009                .await?
1010                {
1011                    fallback::ErrorHandleResult::EarlyReturn { status } => {
1012                        // Task rerouted — clean up tmux session and env secrets
1013                        session::cleanup_session(task_id, &tmux, &tmux_session).await;
1014                        // The fallback handler already writes a sanitized `last_error`
1015                        // into the store for EarlyReturn paths (reroutes / needs_review).
1016                        // Passing the raw agent_err here would record noisy provider
1017                        // payloads (e.g. full api_retry NDJSON). Leave error_override
1018                        // as None so build_run_audit will prefer the store's last_error
1019                        // (see src/engine/runner/fallback.rs which persists a compact
1020                        // summary for rate-limit and reroute cases).
1021                        let audit = self
1022                            .build_run_audit(RunAuditInput {
1023                                task_id,
1024                                status: &status,
1025                                parse_result: &parse_result,
1026                                raw_stdout: &session_output.raw_stdout,
1027                                raw_stderr: &session_output.raw_stderr,
1028                                started_at,
1029                                error_override: None,
1030                                elapsed_secs: session_output.elapsed_secs,
1031                                push_failed: false,
1032                            })
1033                            .await;
1034                        return Ok(Some(RunExecution {
1035                            status,
1036                            exit_code: Some(session_output.exit_code),
1037                            audit,
1038                        }));
1039                    }
1040                    fallback::ErrorHandleResult::Continue { status, error } => {
1041                        (status, false, Some(error))
1042                    }
1043                }
1044            }
1045        };
1046
1047        // Kill tmux session if still alive
1048        session::cleanup_session(task_id, &tmux, &tmux_session).await;
1049
1050        // For no-op done tasks (agent finished with no code changes and no PR),
1051        // clean up the worktree and branch immediately — tmux session is already
1052        // dead so the tmux guard won't block removal.
1053        if final_status.as_str() == "done" {
1054            if let Some(ref st) = self.store {
1055                if let Err(e) =
1056                    crate::engine::cleanup::cleanup_task_worktree(task_id, &self.repo, st).await
1057                {
1058                    tracing::warn!(task_id, error = ?e, "worktree cleanup failed after no-op done");
1059                }
1060            }
1061        }
1062
1063        // Return final status and the session exit code so callers can record it.
1064        // Use `fallback_error` as error_override when available: it carries the error message
1065        // attributed to the current agent (set by handle_error), preventing stale last_error
1066        // values from a prior run from contaminating this run's audit record.
1067        let audit = self
1068            .build_run_audit(RunAuditInput {
1069                task_id,
1070                status: &final_status,
1071                parse_result: &parse_result,
1072                raw_stdout: &session_output.raw_stdout,
1073                raw_stderr: &session_output.raw_stderr,
1074                started_at,
1075                error_override: fallback_error,
1076                elapsed_secs: session_output.elapsed_secs,
1077                push_failed,
1078            })
1079            .await;
1080
1081        Ok(Some(RunExecution {
1082            status: final_status.to_string(),
1083            exit_code: Some(session_output.exit_code),
1084            audit,
1085        }))
1086    }
1087
1088    /// Record task execution metrics to the database.
1089    #[allow(clippy::too_many_arguments)]
1090    async fn record_metrics(
1091        &self,
1092        task_id: &str,
1093        agent_name: &str,
1094        model_name: &Option<String>,
1095        route_result: &Option<RouteResult>,
1096        started_at: &chrono::DateTime<Utc>,
1097        attempts: u32,
1098        final_status: &str,
1099        error_type: Option<&str>,
1100    ) {
1101        let completed_at = Utc::now();
1102        let duration_seconds = (completed_at - *started_at).num_milliseconds() as f64 / 1000.0;
1103
1104        // Classify outcome based on the final status and optional error_type/last_error
1105        let outcome = match final_status {
1106            "done" | "in_progress" | "in_review" => "success",
1107            "new" => "rerouted",
1108            "needs_review" => classify_run_error_type(error_type.unwrap_or("")),
1109            _ => "unknown",
1110        };
1111
1112        let complexity = route_result.as_ref().map(|r| r.complexity.clone());
1113        let files_changed = git_ops::count_changed_files(&PathBuf::from(
1114            self.get_field(task_id, "worktree")
1115                .await
1116                .unwrap_or_default(),
1117        ))
1118        .await
1119        .unwrap_or(0);
1120
1121        if let Some(ref store) = self.store {
1122            // Only set error_type for non-success outcomes
1123            let db_error_type: Option<String> = if outcome == "success" {
1124                None
1125            } else {
1126                Some(outcome.to_string())
1127            };
1128
1129            // Read cost data from store
1130            let usage = store::get_token_usage(&self.store, &self.repo, task_id).await;
1131            let cost = store::get_cost_estimate(&self.store, &self.repo, task_id).await;
1132            let input_tokens = if usage.input_tokens > 0 {
1133                Some(usage.input_tokens as i64)
1134            } else {
1135                None
1136            };
1137            let output_tokens = if usage.output_tokens > 0 {
1138                Some(usage.output_tokens as i64)
1139            } else {
1140                None
1141            };
1142            let input_cost = if cost.input_cost_usd > 0.0 {
1143                Some(cost.input_cost_usd)
1144            } else {
1145                None
1146            };
1147            let output_cost = if cost.output_cost_usd > 0.0 {
1148                Some(cost.output_cost_usd)
1149            } else {
1150                None
1151            };
1152            let total_cost = if cost.total_cost_usd > 0.0 {
1153                Some(cost.total_cost_usd)
1154            } else {
1155                None
1156            };
1157
1158            let metric = InsertTaskMetric {
1159                repo: &self.repo,
1160                task_id,
1161                agent: agent_name,
1162                model: model_name.as_deref(),
1163                complexity: complexity.as_deref(),
1164                outcome,
1165                duration_seconds,
1166                started_at,
1167                completed_at: &completed_at,
1168                attempts: attempts as i32 + 1,
1169                files_changed: files_changed as i32,
1170                error_type: db_error_type.as_deref(),
1171                input_tokens,
1172                output_tokens,
1173                input_cost_usd: input_cost,
1174                output_cost_usd: output_cost,
1175                total_cost_usd: total_cost,
1176            };
1177
1178            if let Err(e) = store.insert_task_metric(&metric).await {
1179                tracing::error!(task_id, ?e, "failed to record task metrics");
1180            }
1181        }
1182    }
1183
1184    /// Run a task with full engine context (backend, tmux, capture).
1185    ///
1186    /// Called by the engine dispatch loop with richer context.
1187    /// Returns a `WeightSignal` for the engine to feed back to the router.
1188    pub async fn run_with_context(
1189        &self,
1190        task: &ExternalTask,
1191        backend: &Arc<dyn ExternalBackend>,
1192        _tmux: &Arc<TmuxManager>,
1193        route_result: Option<&RouteResult>,
1194    ) -> anyhow::Result<WeightSignal> {
1195        let task_id = &task.id.0;
1196        let agent = route_result.map(|r| r.agent.as_str());
1197        let agent_name = agent.unwrap_or("claude").to_string();
1198        let model = route_result.and_then(|r| r.model.as_deref());
1199
1200        // Record start time for metrics (before any work begins)
1201        let started_at = Utc::now();
1202
1203        // Store task info for prompt building (title/body already in store tasks table)
1204
1205        // Record run start in task_runs audit trail
1206        let run_audit_id = if let Some(ref store) = self.store {
1207            let attempt: i32 = self
1208                .get_field(task_id, "attempts")
1209                .await
1210                .and_then(|s| s.parse().ok())
1211                .unwrap_or(0)
1212                + 1;
1213            if let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, task_id).await {
1214                match store
1215                    .start_run(&crate::store::StartRun {
1216                        task_id: store_id,
1217                        attempt,
1218                        run_type: "agent",
1219                        agent: &agent_name,
1220                        model: model.unwrap_or(""),
1221                        command: &format!("{} --model {}", agent_name, model.unwrap_or("default")),
1222                        prompt: &task.body,
1223                    })
1224                    .await
1225                {
1226                    Ok(run_id) => Some(run_id),
1227                    Err(e) => {
1228                        tracing::warn!(
1229                            task_id,
1230                            error = %e,
1231                            "failed to record run start in audit trail"
1232                        );
1233                        None
1234                    }
1235                }
1236            } else {
1237                None
1238            }
1239        } else {
1240            None
1241        };
1242
1243        // Run the task.
1244        // Always finalize the run audit row, even on early-return/error paths.
1245        let run_result = match self
1246            .run(task_id, agent, model, Some(&**backend), &started_at)
1247            .await
1248        {
1249            Ok(run_result) => run_result,
1250            Err(e) => {
1251                if let Some(run_id) = run_audit_id {
1252                    if let Some(ref store) = self.store {
1253                        if let Err(complete_err) = store
1254                            .complete_run(&crate::store::CompleteRun {
1255                                run_id,
1256                                exit_code: Some(-1),
1257                                stdout: "",
1258                                stderr: "",
1259                                parsed: "",
1260                                outcome: "failed",
1261                                error: &e.to_string(),
1262                                tokens: crate::store::RunTokenUsage::default(),
1263                            })
1264                            .await
1265                        {
1266                            tracing::warn!(
1267                                task_id,
1268                                run_id,
1269                                error = %complete_err,
1270                                "failed to record run completion in audit trail after runner error"
1271                            );
1272                        }
1273                    }
1274                }
1275                return Err(e);
1276            }
1277        };
1278
1279        // If the runner guard skipped the task, do not re-post stale data as a new comment.
1280        let (status, exit_code_opt, run_audit) = match run_result {
1281            Some(result) => (result.status, result.exit_code, result.audit),
1282            None => {
1283                if let Some(run_id) = run_audit_id {
1284                    if let Some(ref store) = self.store {
1285                        if let Err(e) = store
1286                            .complete_run(&crate::store::CompleteRun {
1287                                run_id,
1288                                exit_code: Some(-1),
1289                                stdout: "",
1290                                stderr: "",
1291                                parsed: "",
1292                                outcome: "aborted",
1293                                error: "run skipped by guard",
1294                                tokens: crate::store::RunTokenUsage::default(),
1295                            })
1296                            .await
1297                        {
1298                            tracing::warn!(
1299                                task_id,
1300                                run_id,
1301                                error = %e,
1302                                "failed to record skipped run completion in audit trail"
1303                            );
1304                        }
1305                    }
1306                }
1307                tracing::info!(task_id, "guard skipped task — not posting stale result");
1308                return Ok(WeightSignal::None);
1309            }
1310        };
1311
1312        // Process delegations if the agent requested subtasks
1313        let delegations_raw = self
1314            .get_field(task_id, "delegations")
1315            .await
1316            .unwrap_or_default();
1317        if !delegations_raw.is_empty() {
1318            match serde_json::from_str::<Vec<crate::parser::Delegation>>(&delegations_raw) {
1319                Ok(delegations) if !delegations.is_empty() => {
1320                    self.process_delegations(task, &delegations, backend)
1321                        .await?;
1322                    if let Err(e) = store::store_set_result(
1323                        &self.store,
1324                        &self.repo,
1325                        task_id,
1326                        &[("delegations", serde_json::json!([]))],
1327                    )
1328                    .await
1329                    {
1330                        tracing::warn!(task_id, err = %e, "failed to clear delegations in store");
1331                    }
1332                }
1333                Ok(_) => {} // empty list, nothing to do
1334                Err(e) => {
1335                    tracing::error!(task_id, error = %e, "corrupt delegations JSON — clearing");
1336                    if let Err(e2) = store::store_set_result(
1337                        &self.store,
1338                        &self.repo,
1339                        task_id,
1340                        &[("delegations", serde_json::json!([]))],
1341                    )
1342                    .await
1343                    {
1344                        tracing::warn!(task_id, err = %e2, "failed to clear corrupt delegations in store");
1345                    }
1346                    if let Err(e2) = store::store_set_result(
1347                        &self.store,
1348                        &self.repo,
1349                        task_id,
1350                        &[(
1351                            "last_error",
1352                            serde_json::json!(format!("delegation parse failed: {e}")),
1353                        )],
1354                    )
1355                    .await
1356                    {
1357                        tracing::warn!(task_id, err = %e2, "failed to write delegation_parse_failed last_error to store");
1358                    }
1359                }
1360            }
1361        }
1362
1363        // Post result to GitHub
1364        let summary = self.get_field(task_id, "summary").await.unwrap_or_default();
1365        let last_error = self
1366            .get_field(task_id, "last_error")
1367            .await
1368            .unwrap_or_default();
1369
1370        // Determine weight signal based on outcome.
1371        // Any reroute ("new"/"routed") should avoid review-gate and go back through
1372        // normal dispatch, regardless of why it was rerouted.
1373        let is_rerouted = status == "new" || status == "routed";
1374        if is_rerouted {
1375            // Check if silence detection (tick phase1b) already re-routed this task while
1376            // the session was running. Silence detection sets status to Routed/New and
1377            // stores the fallback agent — we must NOT overwrite those fields here.
1378            // If the stored status is still InProgress, silence detection did not fire
1379            // and we can safely clear agent/model for normal reroute.
1380            let silence_already_reset = store::opt_store_get_task(&self.store, &self.repo, task_id)
1381                .await
1382                .map(|t| !matches!(t.status, crate::store::TaskStatus::InProgress))
1383                .unwrap_or(false);
1384
1385            if !silence_already_reset {
1386                if let Err(e) = store::store_set_result(
1387                    &self.store,
1388                    &self.repo,
1389                    task_id,
1390                    &[
1391                        ("agent", serde_json::json!("")),
1392                        ("model", serde_json::json!("")),
1393                    ],
1394                )
1395                .await
1396                {
1397                    tracing::warn!(task_id, err = %e, "failed to clear agent/model for reroute in store");
1398                }
1399            } else {
1400                tracing::debug!(
1401                    task_id,
1402                    "silence detection already set fallback agent — skipping agent/model clear"
1403                );
1404            }
1405        }
1406        // Determine weight signal based on outcome.
1407        // Reroutes should only emit RateLimited when the underlying error is genuinely
1408        // a rate limit (not silence detection, timeouts, parse errors, etc.). This prevents
1409        // false cooldown cascades where non-rate-limit failures are treated as rate limits.
1410        let is_rate_limit_error = classify_run_error_type(&last_error) == "rate_limit";
1411        let weight_signal = if is_rerouted {
1412            if is_rate_limit_error {
1413                WeightSignal::RateLimited {
1414                    agent: agent_name.clone(),
1415                }
1416            } else {
1417                // Non-rate-limit reroutes (silence detection, timeout, parse error): signal
1418                // the engine to keep the task in "new" for re-routing. Using Rerouted (not
1419                // None) prevents tick from converting the silence-reset back to needs_review,
1420                // which would trigger the no-code review shortcut and falsely mark it done.
1421                WeightSignal::Rerouted
1422            }
1423        } else if status == "needs_review" {
1424            // Rate limit errors that escalate to needs_review must NOT reset backoff
1425            // counters (issue #2153).  These errors indicate the agent is genuinely
1426            // unavailable — resetting would wipe the cooldown and cause the next dispatch
1427            // to retry the same rate-limited agent immediately.
1428            //
1429            // Detection: classify_run_error_type() confirms "rate limit" in last_error.
1430            // Reuse the same classifier as `is_rate_limit_error` above rather than a raw
1431            // substring check — billing-cycle exhaustion messages (e.g. Kimi's "usage
1432            // limit for this billing cycle") also contain "usage limit"/"rate limit", so
1433            // a bare substring check would record a rate-limit event into the router's
1434            // health signal for a condition that isn't a genuine rate limit (issue #3529).
1435            let has_rate_limit_error = classify_run_error_type(&last_error) == "rate_limit";
1436            if has_rate_limit_error {
1437                // Record in the metrics store so the router's health check detects it.
1438                if let Some(ref store) = self.store {
1439                    if let Err(e) = store
1440                        .record_rate_limit(&agent_name, "rate", Some(task_id))
1441                        .await
1442                    {
1443                        tracing::warn!(task_id, agent = %agent_name, err = %e, "failed to persist rate_limit event — router health check may miss this signal");
1444                    }
1445                }
1446                // Also record via the cooldown system so model_for_complexity() skips
1447                // this model.  Use the stored last_error text as the message since
1448                // classify_run_error_type() already confirmed it contains "rate limit".
1449                crate::engine::cooldown::record_rate_limit(&agent_name, model, &last_error).await;
1450                WeightSignal::RateLimited {
1451                    agent: agent_name.clone(),
1452                }
1453            } else {
1454                // Non-rate-limit "needs_review" (e.g. agent successfully completed but
1455                // review agent is needed) — reset backoff so the next failure starts fresh.
1456                crate::engine::cooldown::record_agent_success(&agent_name, model.unwrap_or(""))
1457                    .await;
1458                WeightSignal::Success {
1459                    agent: agent_name.clone(),
1460                }
1461            }
1462        } else if status == "done" || status == "in_progress" || status == "in_review" {
1463            // Reset exponential backoff counters so the next failure starts fresh.
1464            crate::engine::cooldown::record_agent_success(&agent_name, model.unwrap_or("")).await;
1465            WeightSignal::Success {
1466                agent: agent_name.clone(),
1467            }
1468        } else if status == "blocked" {
1469            WeightSignal::Blocked
1470        } else {
1471            WeightSignal::None
1472        };
1473
1474        // Record metrics
1475        {
1476            let attempts: u32 = self
1477                .get_field(task_id, "attempts")
1478                .await
1479                .and_then(|s| s.parse().ok())
1480                .unwrap_or(0);
1481            self.record_metrics(
1482                task_id,
1483                &agent_name,
1484                &model.map(|s| s.to_string()),
1485                &route_result.cloned(),
1486                &started_at,
1487                attempts,
1488                &status,
1489                if last_error.is_empty() {
1490                    None
1491                } else {
1492                    Some(&last_error)
1493                },
1494            )
1495            .await;
1496        }
1497
1498        // Complete run in task_runs audit trail (include exit code from runner when available)
1499        if let Some(run_id) = run_audit_id {
1500            if let Some(ref store) = self.store {
1501                if let Err(e) = store
1502                    .complete_run(&crate::store::CompleteRun {
1503                        run_id,
1504                        exit_code: exit_code_opt,
1505                        stdout: &run_audit.stdout,
1506                        stderr: &run_audit.stderr,
1507                        parsed: &run_audit.parsed_response,
1508                        outcome: &run_audit.outcome,
1509                        error: &run_audit.error,
1510                        tokens: crate::store::RunTokenUsage {
1511                            input_tokens: run_audit.input_tokens,
1512                            output_tokens: run_audit.output_tokens,
1513                            total_cost_usd: run_audit.total_cost_usd,
1514                            duration_secs: run_audit.duration_secs,
1515                        },
1516                    })
1517                    .await
1518                {
1519                    tracing::warn!(
1520                        task_id,
1521                        run_id,
1522                        error = %e,
1523                        "failed to record run completion in audit trail"
1524                    );
1525                }
1526            }
1527        }
1528
1529        // If task was rerouted (status=routed or new after run), update GitHub agent label
1530        // so the router doesn't re-route back to the same failed agent.
1531        if is_rerouted && !is_internal_id(task_id) {
1532            let new_agent = self.get_field(task_id, "agent").await.unwrap_or_default();
1533            if !new_agent.is_empty() && new_agent != agent_name {
1534                update_agent_label_after_reroute(
1535                    &self.repo,
1536                    task,
1537                    backend,
1538                    &agent_name,
1539                    &new_agent,
1540                )
1541                .await;
1542            }
1543        }
1544
1545        // Internal tasks: skip GitHub status/comment updates — dispatch phase handles status.
1546        if is_internal_id(task_id) {
1547            return Ok(weight_signal);
1548        }
1549
1550        // Update GitHub status
1551        let new_status = match status.as_str() {
1552            "done" => Status::Done,
1553            "in_progress" => Status::InProgress,
1554            "in_review" => Status::InReview,
1555            "blocked" => Status::Blocked,
1556            "needs_review" => Status::NeedsReview,
1557            "routed" => Status::Routed, // Rerouted (agent/model already set)
1558            "new" => Status::New,       // Legacy reroute path
1559            _ => Status::NeedsReview,
1560        };
1561        // Store-first: update SQLite (must succeed), then mirror to backend.
1562        if let Some(ref store) = self.store {
1563            if let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &task.id.0).await {
1564                store
1565                    .update_status(
1566                        store_id,
1567                        crate::engine::tasks::status_to_task_status(new_status),
1568                    )
1569                    .await
1570                    .context("store-first status update failed in runner")?;
1571            }
1572        }
1573        if let Err(e) = backend.update_status(&task.id, new_status).await {
1574            tracing::warn!(
1575                task_id,
1576                ?new_status,
1577                err = %e,
1578                "failed to mirror runner status to backend — store is authoritative"
1579            );
1580        }
1581
1582        let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ");
1583        let mut comment = if !summary.is_empty() {
1584            format!("[{now}] {status}: {summary}")
1585        } else if !last_error.is_empty() {
1586            format!("[{now}] {status}: {last_error}")
1587        } else {
1588            format!("[{now}] {status}")
1589        };
1590
1591        // Append visible attribution footer (matches PR body style)
1592        let footer = crate::engine::attribution_footer("Created", &agent_name, model);
1593        comment.push_str(&footer);
1594
1595        backend.post_comment(&task.id, &comment).await?;
1596
1597        Ok(weight_signal)
1598    }
1599
1600    /// Process delegations from an agent response.
1601    ///
1602    /// Creates child GitHub issues for each delegation and marks the parent
1603    /// as blocked. The engine's Phase 4 unblock mechanism will re-activate
1604    /// the parent when all children are done.
1605    async fn process_delegations(
1606        &self,
1607        parent_task: &ExternalTask,
1608        delegations: &[crate::parser::Delegation],
1609        backend: &Arc<dyn ExternalBackend>,
1610    ) -> anyhow::Result<()> {
1611        let parent_id = &parent_task.id;
1612
1613        // Fetch existing open tasks once for dedup — prevents creating duplicate
1614        // GitHub issues when the same delegations are produced on repeated runs.
1615        let existing_titles: std::collections::HashSet<String> =
1616            match backend.list_all_tasks().await {
1617                Ok(tasks) => tasks.into_iter().map(|t| t.title).collect(),
1618                Err(e) => {
1619                    tracing::warn!(err = %e, "failed to fetch tasks for dedup — will create all");
1620                    std::collections::HashSet::new()
1621                }
1622            };
1623
1624        for delegation in delegations {
1625            // Skip if an issue with the same title already exists
1626            if existing_titles.contains(&delegation.title) {
1627                tracing::info!(
1628                    parent = parent_id.0,
1629                    title = delegation.title,
1630                    "skipping delegation — issue with same title already exists"
1631                );
1632                continue;
1633            }
1634
1635            // Build labels: status:new + any labels from the delegation
1636            let mut labels = delegation.labels.clone();
1637            labels.push("status:new".to_string());
1638
1639            // Build child body with delegation reference
1640            let child_body = format!(
1641                "{}\n\n---\n_Delegated from #{}_{}",
1642                delegation.body,
1643                parent_id.0,
1644                crate::engine::orch_footer()
1645            );
1646
1647            match backend
1648                .create_sub_task(parent_id, &delegation.title, &child_body, &labels)
1649                .await
1650            {
1651                Ok(child_id) => {
1652                    tracing::info!(
1653                        parent = parent_id.0,
1654                        child = child_id.0,
1655                        title = delegation.title,
1656                        "created delegated subtask"
1657                    );
1658                }
1659                Err(e) => {
1660                    tracing::error!(
1661                        parent = parent_id.0,
1662                        title = delegation.title,
1663                        err = %e,
1664                        "failed to create delegated subtask"
1665                    );
1666                }
1667            }
1668        }
1669
1670        // Mark parent as blocked — store-first, then mirror to backend.
1671        if let Some(ref store) = self.store {
1672            if let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &parent_id.0).await {
1673                store
1674                    .update_status(store_id, crate::store::TaskStatus::Blocked)
1675                    .await
1676                    .context("store-first: failed to block parent in store")?;
1677            }
1678        }
1679        if let Err(e) = backend.update_status(parent_id, Status::Blocked).await {
1680            tracing::warn!(
1681                parent = parent_id.0,
1682                err = %e,
1683                "failed to mirror blocked status to backend — store is authoritative"
1684            );
1685        }
1686
1687        // Post summary comment on parent
1688        let summary = delegations
1689            .iter()
1690            .enumerate()
1691            .map(|(i, d)| format!("{}. {}", i + 1, d.title))
1692            .collect::<Vec<_>>()
1693            .join("\n");
1694
1695        backend
1696            .post_comment(
1697                parent_id,
1698                &format!(
1699                    "Delegated {} subtask(s):\n\n{}\n\nParent task is blocked until all subtasks complete.{}",
1700                    delegations.len(),
1701                    summary,
1702                    crate::engine::orch_footer()
1703                ),
1704            )
1705            .await?;
1706
1707        Ok(())
1708    }
1709
1710    /// Resolve the project directory for this repo.
1711    async fn resolve_project_dir(&self) -> anyhow::Result<PathBuf> {
1712        // Explicit env var always wins
1713        if let Ok(dir) = std::env::var("PROJECT_DIR") {
1714            if !dir.is_empty() {
1715                return Ok(PathBuf::from(dir));
1716            }
1717        }
1718
1719        // Check projects list in global config — find path for this repo
1720        if let Ok(paths) = config::get_project_paths() {
1721            for path_str in &paths {
1722                let path = PathBuf::from(path_str);
1723                // Check if this project's .orch.yml has matching repo
1724                if let Ok(repo) = config::get_repo_for_project(&path) {
1725                    // Use tokio::fs::metadata for async-safe existence check
1726                    let path_for_check = path.clone();
1727                    if repo == self.repo && tokio::fs::metadata(&path_for_check).await.is_ok() {
1728                        return Ok(path);
1729                    }
1730                }
1731            }
1732        }
1733
1734        // Legacy: check config project_dir
1735        if let Ok(dir) = config::get("project_dir") {
1736            if !dir.is_empty() {
1737                let path = PathBuf::from(&dir);
1738                let path_for_check = path.clone();
1739                if tokio::fs::metadata(&path_for_check).await.is_ok() {
1740                    return Ok(path);
1741                }
1742            }
1743        }
1744
1745        // Check for bare clone
1746        let parts: Vec<&str> = self.repo.split('/').collect();
1747        if parts.len() == 2 {
1748            let bare = self
1749                .orch_home
1750                .join("projects")
1751                .join(parts[0])
1752                .join(format!("{}.git", parts[1]));
1753            let bare_for_check = bare.clone();
1754            if tokio::task::spawn_blocking(move || bare_for_check.exists())
1755                .await
1756                .unwrap_or(false)
1757            {
1758                return Ok(bare);
1759            }
1760        }
1761
1762        // No resolvable project directory: not in active config, no legacy
1763        // project_dir, no bare clone. Falling back to the service's own CWD
1764        // here would silently point worktree creation at an unrelated
1765        // directory (see issue #3416) — surface a clear error instead.
1766        anyhow::bail!(
1767            "no resolvable project directory for repo {} — not in active config, no bare clone at {}, no legacy project_dir set",
1768            self.repo,
1769            self.orch_home
1770                .join("projects")
1771                .join(self.repo.split('/').next().unwrap_or_default())
1772                .join(format!(
1773                    "{}.git",
1774                    self.repo.split('/').nth(1).unwrap_or_default()
1775                ))
1776                .display()
1777        )
1778    }
1779}
1780
1781/// Update the GitHub agent label on an external task when it is rerouted to a new agent.
1782///
1783/// Removes the old `agent:<old_agent>` label, ensures the new `agent:<new_agent>` label exists
1784/// in the repo, then applies it. Returns `true` only when both `ensure_label` (repo label creation)
1785/// and `set_labels` (issue label assignment) succeed. `remove_label` failures are non-fatal.
1786async fn update_agent_label_after_reroute(
1787    #[cfg_attr(test, allow(unused_variables))] repo: &str,
1788    task: &ExternalTask,
1789    backend: &Arc<dyn ExternalBackend>,
1790    old_agent: &str,
1791    new_agent: &str,
1792) -> bool {
1793    let old_label = format!("agent:{old_agent}");
1794    if let Err(e) = backend.remove_label(&task.id, &old_label).await {
1795        tracing::warn!(
1796            task_id = ?task.id,
1797            label = old_label,
1798            error = %e,
1799            "failed to remove old agent label during re-route"
1800        );
1801    }
1802    let new_label = format!("agent:{new_agent}");
1803
1804    // ensure_label creates the repo-level label if missing (best-effort).
1805    // Skipped under #[cfg(test)] to avoid real GitHub API calls.
1806    #[cfg(not(test))]
1807    {
1808        match crate::github::http::GhHttp::new() {
1809            Ok(gh) => {
1810                if gh
1811                    .ensure_label(
1812                        repo,
1813                        &new_label,
1814                        crate::github::http::status_label_color(&new_label),
1815                        &format!("Agent: {new_agent}"),
1816                    )
1817                    .await
1818                    .is_err()
1819                {
1820                    tracing::warn!(
1821                        task_id = ?task.id,
1822                        label = %new_label,
1823                        "ensure_label failed during agent label update after failover"
1824                    );
1825                }
1826            }
1827            Err(e) => {
1828                tracing::warn!(
1829                    task_id = ?task.id,
1830                    err = %e,
1831                    "GhHttp::new() failed — skipping ensure_label"
1832                );
1833            }
1834        }
1835    }
1836
1837    if let Err(e) = backend
1838        .set_labels(&task.id, std::slice::from_ref(&new_label))
1839        .await
1840    {
1841        tracing::warn!(
1842            task_id = ?task.id,
1843            label = %new_label,
1844            error = %e,
1845            "set_labels failed during agent label update after failover"
1846        );
1847        return false;
1848    }
1849    true
1850}
1851
1852#[cfg(test)]
1853mod tests {
1854    use super::*;
1855    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
1856    use crate::engine::runner::agents::patterns::safe_tail as safe_utf8_tail;
1857    use crate::parser::AgentResponse;
1858    use crate::store::{NewTask, TaskStore};
1859    use async_trait::async_trait;
1860    use chrono::Utc;
1861    use once_cell::sync::Lazy;
1862    use std::sync::{Arc, Mutex};
1863    use tempfile::TempDir;
1864
1865    static ENV_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
1866
1867    fn ok_parse_result(status: &str) -> Result<agents::ParsedResponse, agents::AgentError> {
1868        Ok(agents::ParsedResponse {
1869            response: AgentResponse {
1870                status: status.to_string(),
1871                summary: String::new(),
1872                accomplished: vec![],
1873                remaining: vec![],
1874                files: vec![],
1875                error: None,
1876                input_tokens: None,
1877                output_tokens: None,
1878                learnings: vec![],
1879                delegations: vec![],
1880            },
1881            input_tokens: None,
1882            output_tokens: None,
1883            duration_ms: None,
1884        })
1885    }
1886
1887    // ── safe_utf8_tail ───────────────────────────────────────────────────────
1888
1889    #[test]
1890    fn safe_utf8_tail_short_string_returned_as_is() {
1891        assert_eq!(safe_utf8_tail("hello", 100), "hello");
1892        assert_eq!(safe_utf8_tail("", 10), "");
1893    }
1894
1895    #[test]
1896    fn safe_utf8_tail_truncates_ascii() {
1897        let s = "abcdefghij"; // 10 bytes
1898        assert_eq!(safe_utf8_tail(s, 5), "fghij");
1899        assert_eq!(safe_utf8_tail(s, 10), "abcdefghij");
1900    }
1901
1902    #[test]
1903    fn safe_utf8_tail_handles_multibyte_boundary() {
1904        // "日本語" is 3 chars × 3 bytes each = 9 bytes total
1905        let s = "日本語";
1906        assert_eq!(s.len(), 9);
1907        // Cutting at 8 bytes lands in the middle of "語" (3 bytes at 6..9).
1908        // safe_utf8_tail should walk forward to the next boundary at 9 (end) rather
1909        // than panicking on an invalid slice.
1910        let tail = safe_utf8_tail(s, 8);
1911        assert!(
1912            s.ends_with(tail),
1913            "tail must be a valid suffix of the original"
1914        );
1915        assert!(
1916            std::str::from_utf8(tail.as_bytes()).is_ok(),
1917            "tail must be valid UTF-8"
1918        );
1919    }
1920
1921    #[test]
1922    fn safe_utf8_tail_handles_mixed_ascii_and_multibyte() {
1923        let s = "hello日本語world"; // ascii + CJK + ascii
1924        for max_bytes in [1, 3, 5, 7, 11, 13, 20, 100] {
1925            let tail = safe_utf8_tail(s, max_bytes);
1926            assert!(
1927                std::str::from_utf8(tail.as_bytes()).is_ok(),
1928                "max_bytes={max_bytes}: tail must be valid UTF-8"
1929            );
1930            assert!(
1931                s.ends_with(tail),
1932                "max_bytes={max_bytes}: tail must be a suffix of original"
1933            );
1934        }
1935    }
1936
1937    #[test]
1938    fn safe_utf8_tail_exact_boundary() {
1939        let s = "abcdef"; // 6 bytes
1940        assert_eq!(safe_utf8_tail(s, 6), "abcdef"); // exactly fits
1941        assert_eq!(safe_utf8_tail(s, 0), ""); // empty tail
1942    }
1943
1944    #[test]
1945    fn parse_success_output_synthesizes_done_from_plain_text() {
1946        let runner = agents::get_runner("claude");
1947        let parsed = parse_success_output(
1948            "123",
1949            "claude",
1950            &*runner,
1951            "No open positions, no trade executions, no conditions change.",
1952        )
1953        .unwrap();
1954
1955        assert_eq!(parsed.response.status, "done");
1956        assert_eq!(
1957            parsed.response.summary,
1958            "No open positions, no trade executions, no conditions change."
1959        );
1960    }
1961
1962    #[test]
1963    fn parse_success_output_synthesizes_needs_review_from_error_text() {
1964        let runner = agents::get_runner("claude");
1965        let parsed =
1966            parse_success_output("123", "claude", &*runner, "Failed to update branch").unwrap();
1967
1968        assert_eq!(parsed.response.status, "needs_review");
1969        assert_eq!(
1970            parsed.response.error.as_deref(),
1971            Some("agent returned plain text instead of JSON")
1972        );
1973    }
1974
1975    #[test]
1976    fn parse_success_output_keeps_empty_text_invalid() {
1977        let runner = agents::get_runner("claude");
1978        let err = parse_success_output("123", "claude", &*runner, "").unwrap_err();
1979
1980        assert!(matches!(err, agents::AgentError::InvalidResponse { .. }));
1981    }
1982
1983    #[test]
1984    fn parse_success_output_preserves_tokens_from_ndjson() {
1985        // Simulates NDJSON output from --output-format stream-json where
1986        // the result event contains token usage. parse_response fails on
1987        // NDJSON (expects single JSON blob), but find_agent_result extracts
1988        // tokens from the NDJSON envelope so cost tracking is preserved.
1989        let ndjson = r#"{"type":"system","subtype":"init","session_id":"abc"}
1990{"type":"assistant","message":{"content":[{"type":"text","text":"Done."}],"usage":{"input_tokens":100,"output_tokens":50}}}
1991{"type":"result","subtype":"success","is_error":false,"duration_ms":5000,"result":"All changes committed successfully","usage":{"input_tokens":1234,"output_tokens":567}}"#;
1992        let runner = agents::get_runner("claude");
1993        let parsed = parse_success_output("456", "claude", &*runner, ndjson).unwrap();
1994
1995        assert_eq!(parsed.response.status, "done");
1996        assert_eq!(parsed.input_tokens, Some(1234));
1997        assert_eq!(parsed.output_tokens, Some(567));
1998        assert_eq!(parsed.duration_ms, Some(5000));
1999    }
2000
2001    #[test]
2002    fn parse_success_output_preserves_tokens_from_minimax_ndjson() {
2003        // MiniMax/Kimi use assistant event fallback when there's no result event.
2004        let ndjson = r#"{"type":"system","subtype":"init"}
2005{"type":"assistant","message":{"content":[{"type":"text","text":"No open positions."}],"usage":{"input_tokens":2000,"output_tokens":300}}}"#;
2006        let runner = agents::get_runner("minimax");
2007        let parsed = parse_success_output("789", "minimax", &*runner, ndjson).unwrap();
2008
2009        assert_eq!(parsed.response.status, "done");
2010        assert_eq!(parsed.input_tokens, Some(2000));
2011        assert_eq!(parsed.output_tokens, Some(300));
2012    }
2013
2014    #[test]
2015    fn classify_run_error_type_recognizes_specific_patterns() {
2016        let cases = [
2017            ("push failed: remote rejected", "push_failed"),
2018            ("No commits between main and branch", "no_commits"),
2019            ("create PR failed: 422", "pr_failed"),
2020            ("invalid response from agent", "parse_error"),
2021            ("parse error: missing json", "parse_error"),
2022            ("exceeded max attempts while rerouting", "max_attempts"),
2023            ("unit test failed", "ci_failure"),
2024            ("build failed: type mismatch", "ci_failure"),
2025            ("lint failed on warning", "ci_failure"),
2026            ("3 tests failed", "ci_failure"),
2027            ("test suite failed: 3 tests failed", "ci_failure"),
2028            (
2029                "claude interrupted: exited without a result envelope",
2030                "interrupted",
2031            ),
2032        ];
2033
2034        for (input, expected) in cases {
2035            assert_eq!(classify_run_error_type(input), expected, "input={input}");
2036        }
2037    }
2038
2039    #[test]
2040    fn classify_run_error_type_falls_back_for_unknown_errors() {
2041        assert_eq!(classify_run_error_type("something unexpected"), "failed");
2042    }
2043
2044    #[test]
2045    fn classify_run_error_type_connection_error_mentioning_pull_request_is_not_pr_failed() {
2046        // A generic GitHub API or connection error whose message happens to contain the
2047        // substring "pull request" must NOT be classified as pr_failed.
2048        assert_eq!(
2049            classify_run_error_type("failed to fetch pull request details from GitHub"),
2050            "failed"
2051        );
2052        assert_eq!(
2053            classify_run_error_type("connection reset while reading pull request data"),
2054            "failed"
2055        );
2056    }
2057
2058    #[test]
2059    fn classify_run_error_type_url_with_test_substring_is_not_ci_failure() {
2060        // A connection error whose URL path contains "/test-org/" must NOT be ci_failure.
2061        assert_eq!(
2062            classify_run_error_type(
2063                "failed to connect to api.github.com/repos/test-org/repo/pulls/123"
2064            ),
2065            "failed"
2066        );
2067        // Bare "test" word in an unrelated error must not trigger ci_failure.
2068        assert_eq!(
2069            classify_run_error_type("context deadline exceeded testing connection"),
2070            "failed"
2071        );
2072    }
2073
2074    #[test]
2075    fn classify_run_error_type_usage_without_limit_is_not_rate_limit() {
2076        // "usage" alone (e.g. token usage reporting) must NOT be rate_limit.
2077        assert_eq!(
2078            classify_run_error_type("token usage: 1245 tokens"),
2079            "failed"
2080        );
2081        assert_eq!(
2082            classify_run_error_type("incorrect usage of git command"),
2083            "failed"
2084        );
2085        // Only "usage limit" should trigger rate_limit.
2086        assert_eq!(
2087            classify_run_error_type("exceeded usage limit for this billing period"),
2088            "rate_limit"
2089        );
2090    }
2091
2092    #[test]
2093    fn classify_run_error_type_billing_cycle_is_not_auth_error() {
2094        // "billing cycle" quota-exhaustion messages must be classified as billing_cycle_exhausted,
2095        // not auth_error — aligning with cooldown.rs CreditExhaustionReason::BillingCycleExhausted.
2096        assert_eq!(
2097            classify_run_error_type(
2098                "You've reached your usage limit for this billing cycle. \
2099                 Your quota will be refreshed in the next cycle."
2100            ),
2101            "billing_cycle_exhausted"
2102        );
2103        assert_eq!(
2104            classify_run_error_type("quota exhausted for billing cycle"),
2105            "billing_cycle_exhausted"
2106        );
2107        // Plain "billing" without "cycle" that used to map to auth_error now falls through to "failed".
2108        assert_eq!(
2109            classify_run_error_type("billing account suspended"),
2110            "failed"
2111        );
2112    }
2113
2114    #[test]
2115    fn classify_run_error_type_monthly_quota_is_billing_cycle() {
2116        // Regression test for issue #3302: GitHub Copilot "You have exceeded your
2117        // monthly quota" must map to billing_cycle_exhausted, not "failed".
2118        assert_eq!(
2119            classify_run_error_type(
2120                "opencode failed: agent failed: You have exceeded your monthly quota"
2121            ),
2122            "billing_cycle_exhausted"
2123        );
2124        assert_eq!(
2125            classify_run_error_type("You have exceeded your monthly quota"),
2126            "billing_cycle_exhausted"
2127        );
2128    }
2129
2130    #[test]
2131    fn classify_run_outcome_routed_is_success() {
2132        // "routed" is a canonical non-success status (task will be re-dispatched).
2133        // It should be classified as "success" so the audit trail distinguishes it
2134        // from genuine failures (#2801).
2135        let parse_result = ok_parse_result("routed");
2136        assert_eq!(
2137            classify_run_outcome("routed", &parse_result, false),
2138            "success"
2139        );
2140    }
2141
2142    #[test]
2143    fn classify_run_outcome_blocked_is_blocked() {
2144        let parse_result = ok_parse_result("blocked");
2145        assert_eq!(
2146            classify_run_outcome("blocked", &parse_result, false),
2147            "blocked"
2148        );
2149    }
2150
2151    #[test]
2152    fn classify_run_outcome_non_canonical_is_failed() {
2153        // Non-canonical statuses should be classified as "failed" in the audit.
2154        // The explicit error message is provided by build_run_audit.
2155        let parse_result = ok_parse_result("fix_deployed");
2156        assert_eq!(
2157            classify_run_outcome("fix_deployed", &parse_result, false),
2158            "failed"
2159        );
2160    }
2161
2162    #[test]
2163    fn classify_run_outcome_kimi_billing_cycle_is_not_rate_limit() {
2164        // Regression test for issue #3529: Kimi's exact billing-cycle exhaustion
2165        // message was stored as outcome="rate_limit" because the generic "usage
2166        // limit" pattern in detect_rate_limit() matched before credit-exhaustion
2167        // detection ever ran. The outcome must reflect the actual failure mode.
2168        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2169            Err(agents::AgentError::RateLimit {
2170                message: "Failed to authenticate. API Error: 403 You've reached your usage \
2171                          limit for this billing cycle. Your quota will be refreshed soon."
2172                    .to_string(),
2173            });
2174        assert_eq!(
2175            classify_run_outcome("needs_review", &parse_result, false),
2176            "billing_cycle_exhausted"
2177        );
2178    }
2179
2180    #[test]
2181    fn classify_run_outcome_generic_rate_limit_is_unaffected() {
2182        // A genuine rate limit (no billing-cycle/credit-exhaustion markers) must
2183        // still classify as "rate_limit".
2184        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2185            Err(agents::AgentError::RateLimit {
2186                message: "429 Too Many Requests".to_string(),
2187            });
2188        assert_eq!(
2189            classify_run_outcome("needs_review", &parse_result, false),
2190            "rate_limit"
2191        );
2192    }
2193
2194    #[test]
2195    fn classify_run_outcome_auth_billing_cycle_is_not_failed() {
2196        // Auth-classified billing-cycle exhaustion (e.g. "billing cycle exhausted"
2197        // matches detect_auth_error's pattern list) must also be distinguished from
2198        // a generic auth failure.
2199        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2200            Err(agents::AgentError::Auth {
2201                message: "billing cycle exhausted for this account".to_string(),
2202            });
2203        assert_eq!(
2204            classify_run_outcome("needs_review", &parse_result, false),
2205            "billing_cycle_exhausted"
2206        );
2207    }
2208
2209    #[test]
2210    fn classify_run_outcome_generic_auth_error_is_unaffected() {
2211        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2212            Err(agents::AgentError::Auth {
2213                message: "401 Unauthorized".to_string(),
2214            });
2215        assert_eq!(
2216            classify_run_outcome("needs_review", &parse_result, false),
2217            "failed"
2218        );
2219    }
2220
2221    #[test]
2222    fn classify_run_outcome_interrupted() {
2223        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2224            Err(agents::AgentError::Interrupted {
2225                message: "exited without a result envelope".to_string(),
2226            });
2227        assert_eq!(
2228            classify_run_outcome("new", &parse_result, false),
2229            "interrupted"
2230        );
2231    }
2232
2233    #[tokio::test]
2234    async fn build_run_audit_success_ignores_stale_last_error() {
2235        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2236        let task_id = store
2237            .create(&NewTask {
2238                repo: "owner/repo".to_string(),
2239                origin: "internal".to_string(),
2240                title: "Test".to_string(),
2241                ..Default::default()
2242            })
2243            .await
2244            .unwrap();
2245        store
2246            .set_fields(
2247                task_id,
2248                &[("last_error", serde_json::json!("no error info available"))],
2249            )
2250            .await
2251            .unwrap();
2252
2253        let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2254        let parse_result = ok_parse_result("done");
2255        let started_at = Utc::now();
2256        let audit = runner
2257            .build_run_audit(RunAuditInput {
2258                task_id: &task_id.to_string(),
2259                status: "done",
2260                parse_result: &parse_result,
2261                raw_stdout: "",
2262                raw_stderr: "",
2263                started_at: &started_at,
2264                error_override: None,
2265                elapsed_secs: Some(1),
2266                push_failed: false,
2267            })
2268            .await;
2269
2270        assert_eq!(audit.outcome, "success");
2271        assert!(audit.error.is_empty());
2272    }
2273
2274    #[tokio::test]
2275    async fn build_run_audit_ok_parse_prefers_parsed_error_over_stale_last_error() {
2276        // needs_review classifies as success and wipes the error. The #3851
2277        // misattribution is an Ok parse whose outcome is failed or blocked
2278        let cases = [("needs_action", "failed"), ("blocked", "blocked")];
2279        let parsed_error = "passbox Touch ID prompt timed out for cf/read-token";
2280        let stale = "minimax auth error: API Error: 402 insufficient balance (1008), clearing agent/model for re-route";
2281        for (status, expected_outcome) in cases {
2282            let store = Arc::new(TaskStore::open_memory().await.unwrap());
2283            let task_id = store
2284                .create(&NewTask {
2285                    repo: "owner/repo".to_string(),
2286                    origin: "internal".to_string(),
2287                    title: "Test".to_string(),
2288                    ..Default::default()
2289                })
2290                .await
2291                .unwrap();
2292            store
2293                .set_fields(task_id, &[("last_error", serde_json::json!(stale))])
2294                .await
2295                .unwrap();
2296
2297            let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2298            let parse_result = Ok(agents::ParsedResponse {
2299                response: AgentResponse {
2300                    status: status.to_string(),
2301                    summary: "cache_stats.py did not run".to_string(),
2302                    accomplished: vec![],
2303                    remaining: vec![],
2304                    files: vec![],
2305                    error: Some(parsed_error.to_string()),
2306                    input_tokens: None,
2307                    output_tokens: None,
2308                    learnings: vec![],
2309                    delegations: vec![],
2310                },
2311                input_tokens: None,
2312                output_tokens: None,
2313                duration_ms: None,
2314            });
2315            let started_at = Utc::now();
2316            let audit = runner
2317                .build_run_audit(RunAuditInput {
2318                    task_id: &task_id.to_string(),
2319                    status,
2320                    parse_result: &parse_result,
2321                    raw_stdout: "",
2322                    raw_stderr: "",
2323                    started_at: &started_at,
2324                    error_override: None,
2325                    elapsed_secs: Some(261),
2326                    push_failed: false,
2327                })
2328                .await;
2329
2330            assert_eq!(audit.outcome, expected_outcome, "status={status}");
2331            assert_eq!(audit.error, parsed_error, "status={status}");
2332        }
2333    }
2334
2335    #[tokio::test]
2336    async fn build_run_audit_non_success_uses_fallback_error_when_empty() {
2337        let runner = TaskRunner::new("owner/repo".to_string());
2338        let parse_result = ok_parse_result("new");
2339        let started_at = Utc::now();
2340        let audit = runner
2341            .build_run_audit(RunAuditInput {
2342                task_id: "1",
2343                status: "new",
2344                parse_result: &parse_result,
2345                raw_stdout: "",
2346                raw_stderr: "",
2347                started_at: &started_at,
2348                error_override: None,
2349                elapsed_secs: Some(1),
2350                push_failed: false,
2351            })
2352            .await;
2353
2354        assert_eq!(audit.outcome, "failed");
2355        // "new" is a canonical non-success status — no error message needed.
2356        assert_eq!(audit.error, "");
2357    }
2358
2359    #[tokio::test]
2360    async fn build_run_audit_non_canonical_status_reports_explicit_error() {
2361        // Non-canonical statuses (not in the allowlist) with no parse error
2362        // should get an explicit "unrecognized status: X" message instead of
2363        // the opaque "no error info available" placeholder (#2801).
2364        let runner = TaskRunner::new("owner/repo".to_string());
2365        let parse_result = ok_parse_result("fix_deployed");
2366        let started_at = Utc::now();
2367        let audit = runner
2368            .build_run_audit(RunAuditInput {
2369                task_id: "1",
2370                status: "fix_deployed",
2371                parse_result: &parse_result,
2372                raw_stdout: "",
2373                raw_stderr: "",
2374                started_at: &started_at,
2375                error_override: None,
2376                elapsed_secs: Some(1),
2377                push_failed: false,
2378            })
2379            .await;
2380
2381        assert_eq!(audit.outcome, "failed");
2382        assert_eq!(audit.error, "unrecognized status: fix_deployed");
2383    }
2384
2385    #[tokio::test]
2386    async fn build_run_audit_canonical_non_success_status_no_error_message() {
2387        // Canonical non-success statuses (e.g. "routed") should not produce
2388        // an error message — they're expected outcomes, not failures.
2389        for status in &["routed"] {
2390            let runner = TaskRunner::new("owner/repo".to_string());
2391            let parse_result = ok_parse_result(status);
2392            let started_at = Utc::now();
2393            let audit = runner
2394                .build_run_audit(RunAuditInput {
2395                    task_id: "1",
2396                    status,
2397                    parse_result: &parse_result,
2398                    raw_stdout: "",
2399                    raw_stderr: "",
2400                    started_at: &started_at,
2401                    error_override: None,
2402                    elapsed_secs: Some(1),
2403                    push_failed: false,
2404                })
2405                .await;
2406
2407            assert_eq!(audit.outcome, "success", "status={status}");
2408            assert_eq!(
2409                audit.error, "",
2410                "status={status}: no error for canonical status"
2411            );
2412        }
2413    }
2414
2415    #[tokio::test]
2416    async fn build_run_audit_blocked_uses_summary_when_error_missing() {
2417        let runner = TaskRunner::new("owner/repo".to_string());
2418        let parse_result = Ok(agents::ParsedResponse {
2419            response: AgentResponse {
2420                status: "blocked".to_string(),
2421                summary: "Could not proceed due to missing credentials".to_string(),
2422                accomplished: vec![],
2423                remaining: vec![],
2424                files: vec![],
2425                error: None,
2426                input_tokens: None,
2427                output_tokens: None,
2428                learnings: vec![],
2429                delegations: vec![],
2430            },
2431            input_tokens: None,
2432            output_tokens: None,
2433            duration_ms: None,
2434        });
2435        let started_at = Utc::now();
2436        let audit = runner
2437            .build_run_audit(RunAuditInput {
2438                task_id: "1",
2439                status: "blocked",
2440                parse_result: &parse_result,
2441                raw_stdout: "",
2442                raw_stderr: "",
2443                started_at: &started_at,
2444                error_override: None,
2445                elapsed_secs: Some(1),
2446                push_failed: false,
2447            })
2448            .await;
2449
2450        assert_eq!(audit.outcome, "blocked");
2451        assert_eq!(audit.error, "Could not proceed due to missing credentials");
2452    }
2453
2454    // ── resolve_project_dir ──────────────────────────────────────────────────
2455
2456    #[tokio::test]
2457    async fn resolve_project_dir_uses_project_dir_env() {
2458        // When PROJECT_DIR env var is set, it should always win.
2459        let runner = TaskRunner::new("owner/repo".to_string());
2460
2461        // Use a temp dir so the path exists
2462        let dir = tempfile::tempdir().unwrap();
2463        let dir_str = dir.path().to_string_lossy().to_string();
2464
2465        // Temporarily set the env var — note: tests run in parallel so we use a
2466        // dedicated key that only this test touches. std::env::set_var is not
2467        // thread-safe in general, but the subsequent call is synchronous and
2468        // isolated to the temp dir created above.
2469        std::env::set_var("PROJECT_DIR", &dir_str);
2470        let result = runner.resolve_project_dir().await;
2471        std::env::remove_var("PROJECT_DIR");
2472
2473        assert!(
2474            result.is_ok(),
2475            "resolve_project_dir should succeed with PROJECT_DIR set"
2476        );
2477        assert_eq!(result.unwrap(), dir.path());
2478    }
2479
2480    #[tokio::test]
2481    async fn resolve_project_dir_empty_project_dir_env_falls_through() {
2482        // An empty PROJECT_DIR should be ignored and fall through to other logic.
2483        let runner = TaskRunner::new("owner/testrepo-nonexistent".to_string());
2484        std::env::set_var("PROJECT_DIR", "");
2485        let result = runner.resolve_project_dir().await;
2486        std::env::remove_var("PROJECT_DIR");
2487
2488        // No active config entry, no legacy project_dir, no bare clone for this
2489        // repo: should return an explicit error rather than falling back to the
2490        // service's own CWD (see issue #3416).
2491        assert!(
2492            result.is_err(),
2493            "resolve_project_dir should error when no project dir can be resolved"
2494        );
2495        let err = result.unwrap_err().to_string();
2496        assert!(
2497            err.contains("owner/testrepo-nonexistent"),
2498            "error should name the unresolvable repo: {err}"
2499        );
2500    }
2501
2502    // ── mock backend for process_delegations ─────────────────────────────────
2503
2504    struct TrackingBackend {
2505        sub_tasks_created: Arc<Mutex<Vec<(String, String)>>>, // (title, parent_id)
2506        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
2507        comments: Arc<Mutex<Vec<(String, String)>>>, // (id, body)
2508    }
2509
2510    impl TrackingBackend {
2511        fn new() -> Self {
2512            Self {
2513                sub_tasks_created: Arc::new(Mutex::new(vec![])),
2514                status_updates: Arc::new(Mutex::new(vec![])),
2515                comments: Arc::new(Mutex::new(vec![])),
2516            }
2517        }
2518    }
2519
2520    #[async_trait]
2521    impl crate::backends::ExternalBackend for TrackingBackend {
2522        fn name(&self) -> &str {
2523            "tracking"
2524        }
2525        async fn create_task(
2526            &self,
2527            _t: &str,
2528            _b: &str,
2529            _l: &[String],
2530        ) -> anyhow::Result<ExternalId> {
2531            Ok(ExternalId("new".to_string()))
2532        }
2533        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2534            Ok(ExternalTask {
2535                id: id.clone(),
2536                title: "".to_string(),
2537                body: "".to_string(),
2538                state: "open".to_string(),
2539                labels: vec![],
2540                author: "".to_string(),
2541                created_at: "2026-01-01T00:00:00Z".to_string(),
2542                updated_at: "2026-01-01T00:00:00Z".to_string(),
2543                url: "".to_string(),
2544            })
2545        }
2546        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
2547            Ok(vec![])
2548        }
2549        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2550            Ok(vec![])
2551        }
2552        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2553            self.comments
2554                .lock()
2555                .unwrap()
2556                .push((id.0.clone(), body.to_string()));
2557            Ok(())
2558        }
2559        async fn set_labels(&self, _id: &ExternalId, _l: &[String]) -> anyhow::Result<()> {
2560            Ok(())
2561        }
2562        async fn remove_label(&self, _id: &ExternalId, _l: &str) -> anyhow::Result<()> {
2563            Ok(())
2564        }
2565        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2566            Ok(vec![])
2567        }
2568        async fn create_sub_task(
2569            &self,
2570            parent: &ExternalId,
2571            title: &str,
2572            _body: &str,
2573            _l: &[String],
2574        ) -> anyhow::Result<ExternalId> {
2575            self.sub_tasks_created
2576                .lock()
2577                .unwrap()
2578                .push((title.to_string(), parent.0.clone()));
2579            Ok(ExternalId(format!("child-{}", title)))
2580        }
2581        async fn ensure_status_label(&self, _l: &str) -> anyhow::Result<()> {
2582            Ok(())
2583        }
2584        async fn has_open_issue_with_title(&self, _t: &str, _l: &str) -> anyhow::Result<bool> {
2585            Ok(false)
2586        }
2587        async fn health_check(&self) -> anyhow::Result<()> {
2588            Ok(())
2589        }
2590        async fn is_pr_merged(&self, _b: &str) -> anyhow::Result<bool> {
2591            Ok(false)
2592        }
2593        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2594            Ok(None)
2595        }
2596        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
2597            Ok(vec![])
2598        }
2599        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2600            self.status_updates
2601                .lock()
2602                .unwrap()
2603                .push((id.0.clone(), status));
2604            Ok(())
2605        }
2606    }
2607
2608    fn make_task(id: &str) -> ExternalTask {
2609        ExternalTask {
2610            id: ExternalId(id.to_string()),
2611            title: format!("Task {id}"),
2612            body: "".to_string(),
2613            state: "open".to_string(),
2614            labels: vec![],
2615            author: "bot".to_string(),
2616            created_at: "2026-01-01T00:00:00Z".to_string(),
2617            updated_at: "2026-01-01T00:00:00Z".to_string(),
2618            url: "".to_string(),
2619        }
2620    }
2621
2622    // ── process_delegations ───────────────────────────────────────────────────
2623
2624    #[allow(clippy::await_holding_lock)]
2625    #[tokio::test]
2626    async fn process_delegations_creates_subtasks_and_blocks_parent() {
2627        let _guard = ENV_LOCK.lock().unwrap();
2628        let temp_home = TempDir::new().unwrap();
2629        let orch_home = temp_home.path().join(".orch");
2630        tokio::fs::create_dir_all(&orch_home).await.unwrap();
2631        let old_orch_home = std::env::var("ORCH_HOME").ok();
2632        std::env::set_var("ORCH_HOME", &orch_home);
2633
2634        let runner = TaskRunner::new("owner/repo".to_string());
2635        let parent = make_task("99");
2636        let backend = Arc::new(TrackingBackend::new());
2637        let backend_dyn: Arc<dyn crate::backends::ExternalBackend> = backend.clone();
2638
2639        let delegations = vec![
2640            crate::parser::Delegation {
2641                title: "Subtask A".to_string(),
2642                body: "Do A".to_string(),
2643                labels: vec!["enhancement".to_string()],
2644            },
2645            crate::parser::Delegation {
2646                title: "Subtask B".to_string(),
2647                body: "Do B".to_string(),
2648                labels: vec![],
2649            },
2650        ];
2651
2652        runner
2653            .process_delegations(&parent, &delegations, &backend_dyn)
2654            .await
2655            .unwrap();
2656
2657        // Two sub-tasks should have been created
2658        let created = backend.sub_tasks_created.lock().unwrap();
2659        assert_eq!(created.len(), 2);
2660        assert_eq!(created[0].1, "99", "parent id should be 99");
2661        assert!(
2662            created.iter().any(|(t, _)| t == "Subtask A"),
2663            "Subtask A should be created"
2664        );
2665        assert!(
2666            created.iter().any(|(t, _)| t == "Subtask B"),
2667            "Subtask B should be created"
2668        );
2669        drop(created);
2670
2671        // Parent should be marked blocked
2672        let updates = backend.status_updates.lock().unwrap();
2673        assert!(
2674            updates
2675                .iter()
2676                .any(|(id, s)| id == "99" && *s == Status::Blocked),
2677            "parent should be blocked"
2678        );
2679        drop(updates);
2680
2681        // A comment summarising the delegations should be posted
2682        let comments = backend.comments.lock().unwrap();
2683        assert!(
2684            !comments.is_empty(),
2685            "a delegation summary comment should be posted"
2686        );
2687        let comment_body = &comments[0].1;
2688        assert!(
2689            comment_body.contains("Subtask A"),
2690            "summary should mention Subtask A"
2691        );
2692        assert!(
2693            comment_body.contains("Subtask B"),
2694            "summary should mention Subtask B"
2695        );
2696
2697        if let Some(old) = old_orch_home {
2698            std::env::set_var("ORCH_HOME", old);
2699        } else {
2700            std::env::remove_var("ORCH_HOME");
2701        }
2702    }
2703
2704    #[allow(clippy::await_holding_lock)]
2705    #[tokio::test]
2706    async fn process_delegations_single_subtask() {
2707        let _guard = ENV_LOCK.lock().unwrap();
2708        let temp_home = TempDir::new().unwrap();
2709        let orch_home = temp_home.path().join(".orch");
2710        tokio::fs::create_dir_all(&orch_home).await.unwrap();
2711        let old_orch_home = std::env::var("ORCH_HOME").ok();
2712        std::env::set_var("ORCH_HOME", &orch_home);
2713
2714        let runner = TaskRunner::new("owner/repo".to_string());
2715        let parent = make_task("101");
2716        let backend = Arc::new(TrackingBackend::new());
2717        let backend_dyn: Arc<dyn crate::backends::ExternalBackend> = backend.clone();
2718
2719        let delegations = vec![crate::parser::Delegation {
2720            title: "Only Child".to_string(),
2721            body: "Do only child".to_string(),
2722            labels: vec!["feature".to_string()],
2723        }];
2724
2725        runner
2726            .process_delegations(&parent, &delegations, &backend_dyn)
2727            .await
2728            .unwrap();
2729
2730        let created = backend.sub_tasks_created.lock().unwrap();
2731        assert_eq!(created.len(), 1);
2732        assert_eq!(created[0].0, "Only Child");
2733        drop(created);
2734
2735        let updates = backend.status_updates.lock().unwrap();
2736        assert!(
2737            updates
2738                .iter()
2739                .any(|(id, s)| id == "101" && *s == Status::Blocked),
2740            "parent should be blocked"
2741        );
2742        drop(updates);
2743
2744        let comments = backend.comments.lock().unwrap();
2745        let body = &comments[0].1;
2746        assert!(
2747            body.contains("1 subtask"),
2748            "comment should count one subtask"
2749        );
2750
2751        if let Some(old) = old_orch_home {
2752            std::env::set_var("ORCH_HOME", old);
2753        } else {
2754            std::env::remove_var("ORCH_HOME");
2755        }
2756    }
2757
2758    // ── weight signal logic ───────────────────────────────────────────────────
2759
2760    /// Test helper that mirrors the weight signal logic from `run_with_context`.
2761    /// Reroutes only emit `RateLimited` when the error type is genuinely "rate_limit".
2762    fn weight_signal_for(status: &str, error_type: &str, agent: &str) -> WeightSignal {
2763        let is_rerouted = status == "new" || status == "routed";
2764        let is_rate_limit_error = error_type == "rate_limit";
2765        if is_rerouted {
2766            if is_rate_limit_error {
2767                WeightSignal::RateLimited {
2768                    agent: agent.to_string(),
2769                }
2770            } else {
2771                WeightSignal::Rerouted
2772            }
2773        } else if status == "done"
2774            || status == "needs_review"
2775            || status == "in_progress"
2776            || status == "in_review"
2777        {
2778            WeightSignal::Success {
2779                agent: agent.to_string(),
2780            }
2781        } else if status == "blocked" {
2782            WeightSignal::Blocked
2783        } else {
2784            WeightSignal::None
2785        }
2786    }
2787
2788    #[test]
2789    fn weight_signal_success_includes_needs_review() {
2790        // The normal happy-path for code tasks: agent reports done, has a PR,
2791        // so handle_success returns "needs_review". This must map to Success.
2792        let signal = weight_signal_for("needs_review", "success", "claude");
2793        assert!(
2794            matches!(signal, WeightSignal::Success { agent } if agent == "claude"),
2795            "needs_review should produce WeightSignal::Success"
2796        );
2797    }
2798
2799    #[test]
2800    fn weight_signal_success_for_all_done_statuses() {
2801        for status in ["done", "needs_review", "in_progress", "in_review"] {
2802            let signal = weight_signal_for(status, "success", "codex");
2803            assert!(
2804                matches!(signal, WeightSignal::Success { agent } if agent == "codex"),
2805                "{status} should produce WeightSignal::Success"
2806            );
2807        }
2808    }
2809
2810    #[test]
2811    fn weight_signal_rate_limited_for_reroute_with_rate_limit_error() {
2812        // Reroute due to rate limit should emit RateLimited.
2813        let signal = weight_signal_for("new", "rate_limit", "claude");
2814        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
2815    }
2816
2817    #[test]
2818    fn weight_signal_rate_limited_for_routed_status_with_rate_limit_error() {
2819        let signal = weight_signal_for("routed", "rate_limit", "claude");
2820        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
2821    }
2822
2823    #[test]
2824    fn weight_signal_rerouted_for_non_rate_limit_reroute() {
2825        // Reroute due to silence detection, timeout, or other non-rate-limit errors
2826        // should emit Rerouted (not RateLimited and not None).
2827        // Rerouted maps to "new" in tick so the task stays retryable and is never
2828        // converted to needs_review → done via the no-code review shortcut.
2829        for error_type in ["timeout", "silence detected", "parse_error", "failed"] {
2830            for status in ["new", "routed"] {
2831                let signal = weight_signal_for(status, error_type, "claude");
2832                assert!(
2833                    matches!(signal, WeightSignal::Rerouted),
2834                    "{status} with error_type={error_type} should produce WeightSignal::Rerouted, not RateLimited or None"
2835                );
2836            }
2837        }
2838    }
2839
2840    #[test]
2841    fn silence_reset_rerouted_never_becomes_needs_review() {
2842        // Regression test for: silence detection sets task to new, runner returns
2843        // WeightSignal::Rerouted, tick must map that to "new" — not "needs_review".
2844        // If it mapped to needs_review the no-code shortcut would close the task as
2845        // done with no successful agent run in the audit trail.
2846        let signal = weight_signal_for("new", "silence detected", "claude");
2847        assert!(
2848            matches!(signal, WeightSignal::Rerouted),
2849            "silence-reset reroute must produce Rerouted, never None/needs_review"
2850        );
2851        // Verify tick mapping: Rerouted => "new" (not needs_review or done)
2852        let display_status = match &signal {
2853            WeightSignal::Rerouted => "new",
2854            WeightSignal::None => "needs_review", // the old (buggy) path
2855            _ => "other",
2856        };
2857        assert_eq!(
2858            display_status, "new",
2859            "Rerouted must map to 'new' in tick, not 'needs_review'"
2860        );
2861    }
2862
2863    #[test]
2864    fn weight_signal_blocked_status() {
2865        let signal = weight_signal_for("blocked", "success", "claude");
2866        assert!(matches!(signal, WeightSignal::Blocked));
2867    }
2868
2869    // ── parse_success_output token extraction ────────────────────────────────
2870
2871    /// Mock runner that always returns InvalidResponse with truncated text,
2872    /// simulating what minimax/codex do when they can't parse structured JSON.
2873    struct FailingMockRunner;
2874
2875    impl agents::AgentRunner for FailingMockRunner {
2876        #[cfg(test)]
2877        fn name(&self) -> &str {
2878            "opencode"
2879        }
2880
2881        fn stream_events(&self, _event: &serde_json::Value) -> Vec<agents::StreamEvent> {
2882            vec![]
2883        }
2884
2885        fn build_command(
2886            &self,
2887            _model: Option<&str>,
2888            _timeout_cmd: &str,
2889            _sys_file: &str,
2890            _msg_file: &str,
2891            _permissions: &agents::PermissionRules,
2892        ) -> String {
2893            String::new()
2894        }
2895
2896        fn parse_response(&self, raw: &str) -> Result<agents::ParsedResponse, agents::AgentError> {
2897            // Simulate what minimax does: truncate to 300 chars
2898            Err(agents::AgentError::InvalidResponse {
2899                raw: raw.chars().take(30).collect(),
2900            })
2901        }
2902
2903        fn extract_text(&self, raw: &str) -> Result<String, agents::AgentError> {
2904            Err(agents::AgentError::InvalidResponse {
2905                raw: raw.to_string(),
2906            })
2907        }
2908
2909        fn classify_error(
2910            &self,
2911            exit_code: i32,
2912            _stdout: &str,
2913            _stderr: &str,
2914        ) -> agents::AgentError {
2915            agents::AgentError::Unknown {
2916                exit_code,
2917                message: "mock error".into(),
2918            }
2919        }
2920
2921        fn router_command(
2922            &self,
2923            _prompt: &str,
2924            _model: Option<&str>,
2925        ) -> anyhow::Result<tokio::process::Command> {
2926            anyhow::bail!("not implemented for mock")
2927        }
2928    }
2929
2930    #[test]
2931    fn parse_success_output_extracts_tokens_from_raw_stdout_not_error_raw() {
2932        // OpenCode NDJSON output with token metadata in step_finish
2933        let ndjson = r#"{"type":"text","timestamp":1,"part":{"type":"text","text":"Here is my response: {\"status\":\"done\",\"summary\":\"fixed\"}}"}}
2934{"type":"step_finish","timestamp":2,"part":{"type":"step-finish","reason":"stop","cost":0,"tokens":{"total":95177,"input":94920,"output":257}}}"#;
2935
2936        let runner = FailingMockRunner;
2937        let result = parse_success_output("1550", "opencode", &runner, ndjson);
2938
2939        // Should succeed (synthesized from text) with tokens preserved
2940        let parsed = result.expect("should synthesize response from text");
2941        assert_eq!(parsed.input_tokens, Some(94920));
2942        assert_eq!(parsed.output_tokens, Some(257));
2943    }
2944
2945    #[test]
2946    fn parse_success_output_no_tokens_when_ndjson_has_none() {
2947        // Plain text with an explicit completion signal but no token metadata.
2948        // Uses "No open positions" which is in the explicit_done list.
2949        let text = "No open positions, no trade executions, no conditions change.";
2950
2951        let runner = FailingMockRunner;
2952        let result = parse_success_output("1551", "opencode", &runner, text);
2953
2954        // Should synthesize a "done" response, but without tokens (no NDJSON envelope).
2955        let parsed = result.expect("should synthesize response from explicit-done text");
2956        assert_eq!(parsed.input_tokens, None);
2957        assert_eq!(parsed.output_tokens, None);
2958    }
2959
2960    #[test]
2961    fn parse_success_output_uses_agent_result_directly() {
2962        // Here we test that if find_agent_result succeeds and returns valid JSON, we use it directly
2963        // WITHOUT calling agent_runner.parse_response at all.
2964        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"status\":\"done\",\"summary\":\"my summary\",\"accomplished\":[],\"remaining\":[],\"files\":[]}"}"#;
2965
2966        // FailingMockRunner always returns InvalidResponse, so if we called it, we'd synthesize "done" instead of getting "my summary".
2967        let runner = FailingMockRunner;
2968        let result = parse_success_output("999", "claude", &runner, ndjson);
2969
2970        let parsed = result.expect("should parse successfully via find_agent_result");
2971        assert_eq!(parsed.response.status, "done");
2972        assert_eq!(parsed.response.summary, "my summary");
2973    }
2974
2975    #[test]
2976    fn parse_success_output_treats_agent_result_is_error_as_authoritative() {
2977        // Here we test that if find_agent_result returns is_error=true, we return AgentFailed
2978        // instead of falling back to synthesis or agent_runner.parse_response.
2979        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"I failed because of reasons"}"#;
2980
2981        let runner = FailingMockRunner;
2982        let result = parse_success_output("999", "claude", &runner, ndjson);
2983
2984        let err = result.expect_err("should return error");
2985        match err {
2986            agents::AgentError::AgentFailed { message } => {
2987                assert_eq!(message, "I failed because of reasons");
2988            }
2989            _ => panic!("expected AgentFailed, got {err:?}"),
2990        }
2991    }
2992
2993    #[test]
2994    fn parse_success_output_classifies_model_unavailable_from_is_error_result() {
2995        // Regression test for issue #3607: the is_error=true NDJSON path
2996        // (used by codex's find_codex_result, which stringifies its own
2997        // classified AgentError into result_text) must re-classify a
2998        // model-not-found message as ModelUnavailable so it gets
2999        // model-scoped cooldown treatment instead of degrading to a
3000        // generic AgentFailed, which triggers an agent-wide cooldown.
3001        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"model unavailable (gpt-5.5): Reconnecting... 2/5 (unexpected status 404 Not Found: The model `gpt-5.5` does not exist or you do not have access to it.)"}"#;
3002
3003        let runner = FailingMockRunner;
3004        let result = parse_success_output("999", "claude", &runner, ndjson);
3005
3006        let err = result.expect_err("should return error");
3007        match err {
3008            agents::AgentError::ModelUnavailable { model, .. } => {
3009                assert_eq!(model, "gpt-5.5");
3010            }
3011            _ => panic!("expected ModelUnavailable, got {err:?}"),
3012        }
3013    }
3014
3015    #[test]
3016    fn nonzero_exit_with_ndjson_success_is_parsed_as_success() {
3017        // Simulate an agent that exits with non-zero code but emits NDJSON result
3018        // with is_error=false and a JSON result payload. The runner should accept
3019        // this as success because the NDJSON result is authoritative.
3020        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1234,"result":"{\"status\":\"done\",\"summary\":\"All good\"}","usage":{"input_tokens":10,"output_tokens":5}}"#;
3021
3022        // Use a runner that would normally fail parsing to ensure we're using
3023        // the find_agent_result path. FailingMockRunner.parse_response returns
3024        // InvalidResponse, but parse_success_output should short-circuit.
3025        let runner = FailingMockRunner;
3026
3027        // Construct a fake SessionOutput
3028        let session_output = session::SessionOutput {
3029            exit_code: 1,
3030            raw_stdout: ndjson.to_string(),
3031            raw_stderr: String::new(),
3032            elapsed_secs: Some(1),
3033            sandbox_error: None,
3034        };
3035
3036        // Call the helper we added via the task runner logic by reusing the
3037        // parse_success_output directly since it encapsulates the same behavior.
3038        let parsed =
3039            parse_success_output("task-ndjson", "claude", &runner, &session_output.raw_stdout)
3040                .expect("should parse NDJSON success");
3041        assert_eq!(parsed.response.status, "done");
3042        assert_eq!(parsed.response.summary, "All good");
3043        assert_eq!(parsed.input_tokens, Some(10));
3044        assert_eq!(parsed.output_tokens, Some(5));
3045    }
3046
3047    // ── agent label update after reroute ─────────────────────────────────────
3048
3049    /// Backend that wraps TrackingBackend but can be configured to fail label ops.
3050    struct LabelFailingBackend {
3051        inner: TrackingBackend,
3052        /// If set, set_labels returns this error.
3053        set_labels_err: Option<String>,
3054    }
3055
3056    impl LabelFailingBackend {
3057        fn new() -> Self {
3058            Self {
3059                inner: TrackingBackend::new(),
3060                set_labels_err: None,
3061            }
3062        }
3063        fn with_set_labels_err(mut self, err: impl Into<String>) -> Self {
3064            self.set_labels_err = Some(err.into());
3065            self
3066        }
3067    }
3068
3069    impl Default for LabelFailingBackend {
3070        fn default() -> Self {
3071            Self::new()
3072        }
3073    }
3074
3075    #[async_trait]
3076    impl crate::backends::ExternalBackend for LabelFailingBackend {
3077        fn name(&self) -> &str {
3078            "label_failing"
3079        }
3080        async fn create_task(
3081            &self,
3082            _t: &str,
3083            _b: &str,
3084            _l: &[String],
3085        ) -> anyhow::Result<ExternalId> {
3086            Ok(ExternalId("new".to_string()))
3087        }
3088        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3089            self.inner.get_task(id).await
3090        }
3091        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
3092            Ok(vec![])
3093        }
3094        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3095            Ok(vec![])
3096        }
3097        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
3098            self.inner.post_comment(id, body).await
3099        }
3100        async fn set_labels(&self, id: &ExternalId, labels: &[String]) -> anyhow::Result<()> {
3101            if let Some(ref err) = self.set_labels_err {
3102                return Err(anyhow::anyhow!("{}", err));
3103            }
3104            self.inner.set_labels(id, labels).await
3105        }
3106        async fn remove_label(&self, id: &ExternalId, label: &str) -> anyhow::Result<()> {
3107            self.inner.remove_label(id, label).await
3108        }
3109        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3110            Ok(vec![])
3111        }
3112        async fn create_sub_task(
3113            &self,
3114            parent: &ExternalId,
3115            title: &str,
3116            body: &str,
3117            labels: &[String],
3118        ) -> anyhow::Result<ExternalId> {
3119            self.inner
3120                .create_sub_task(parent, title, body, labels)
3121                .await
3122        }
3123        async fn ensure_status_label(&self, label: &str) -> anyhow::Result<()> {
3124            self.inner.ensure_status_label(label).await
3125        }
3126        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3127            self.inner.update_status(id, status).await
3128        }
3129        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3130            self.inner.get_authenticated_user().await
3131        }
3132        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
3133            Ok(vec![])
3134        }
3135        async fn health_check(&self) -> anyhow::Result<()> {
3136            self.inner.health_check().await
3137        }
3138    }
3139
3140    /// `update_agent_label_after_reroute` returns `false` when `set_labels` fails.
3141    ///
3142    /// This tests the extracted helper directly, bypassing the full `run_with_context`
3143    /// path (which would fail earlier on worktree setup in a unit-test environment).
3144    #[tokio::test]
3145    async fn update_agent_label_returns_false_on_set_labels_failure() {
3146        let task = ExternalTask {
3147            id: ExternalId("test-reroute-1".to_string()),
3148            title: "Test".to_string(),
3149            body: "body".to_string(),
3150            state: "open".to_string(),
3151            labels: vec![],
3152            author: "bot".to_string(),
3153            created_at: "2026-01-01T00:00:00Z".to_string(),
3154            updated_at: "2026-01-01T00:00:00Z".to_string(),
3155            url: "".to_string(),
3156        };
3157        let backend: Arc<dyn crate::backends::ExternalBackend> =
3158            Arc::new(LabelFailingBackend::new().with_set_labels_err("network error"));
3159
3160        let result =
3161            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
3162                .await;
3163
3164        assert!(
3165            !result,
3166            "update_agent_label_after_reroute should return false when set_labels fails"
3167        );
3168    }
3169
3170    /// `update_agent_label_after_reroute` returns `true` when `set_labels` succeeds.
3171    #[tokio::test]
3172    async fn update_agent_label_returns_true_on_success() {
3173        let task = ExternalTask {
3174            id: ExternalId("test-reroute-2".to_string()),
3175            title: "Test".to_string(),
3176            body: "body".to_string(),
3177            state: "open".to_string(),
3178            labels: vec![],
3179            author: "bot".to_string(),
3180            created_at: "2026-01-01T00:00:00Z".to_string(),
3181            updated_at: "2026-01-01T00:00:00Z".to_string(),
3182            url: "".to_string(),
3183        };
3184        // LabelFailingBackend with no injected error delegates to TrackingBackend (always Ok).
3185        let backend: Arc<dyn crate::backends::ExternalBackend> =
3186            Arc::new(LabelFailingBackend::new());
3187
3188        let result =
3189            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
3190                .await;
3191
3192        // ensure_label is skipped in #[cfg(test)], set_labels succeeds via mock.
3193        assert!(
3194            result,
3195            "update_agent_label_after_reroute should return true when set_labels succeeds"
3196        );
3197    }
3198
3199    // ── NDJSON envelope fallback (issue #3078) ────────────────────────────────
3200
3201    /// Pattern A: agent emits domain JSON as result (no `status` field).
3202    /// NDJSON envelope says is_error=false, subtype=success.
3203    /// Was previously returning InvalidResponse → task re-ran.
3204    #[test]
3205    fn parse_success_output_ndjson_envelope_fallback_domain_json_result() {
3206        let domain_result = r#"{"file_path":"md/journal/2026-05-08-morning-briefing.md","summary":"Morning briefing filed","health":{"status":"ok"},"priorities":[]}"#;
3207        let ndjson = format!(
3208            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{domain_result:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
3209        );
3210
3211        let runner = agents::get_runner("glm");
3212        let parsed = parse_success_output("149200", "glm", &*runner, &ndjson)
3213            .expect("should synthesize done from NDJSON envelope when result is domain JSON");
3214
3215        assert_eq!(parsed.response.status, "done");
3216        assert!(
3217            parsed.response.summary.contains("file_path") || !parsed.response.summary.is_empty(),
3218            "summary should be populated from domain JSON result text"
3219        );
3220        assert_eq!(parsed.input_tokens, Some(3000));
3221        assert_eq!(parsed.output_tokens, Some(400));
3222    }
3223
3224    /// Grok-style envelope: no terminal_reason, subtype=success.
3225    #[test]
3226    fn parse_success_output_envelope_fallback_without_terminal_reason() {
3227        let receipt = r#"{"ok":true,"report":"notes.md","commit":"abc"}"#;
3228        let ndjson = format!(
3229            r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","result":{receipt:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
3230        );
3231        let runner = agents::get_runner("grok");
3232        let parsed = parse_success_output("1", "grok", &*runner, &ndjson).expect("done");
3233        assert_eq!(parsed.response.status, "done");
3234        assert_eq!(parsed.input_tokens, Some(3000));
3235    }
3236
3237    /// Pattern B: agent emits plain prose as result, no completion keywords.
3238    /// NDJSON envelope says is_error=false, subtype=success.
3239    /// Was previously returning InvalidResponse → task re-ran.
3240    #[test]
3241    fn parse_success_output_ndjson_envelope_fallback_plain_prose_result() {
3242        let prose = "PR #1008 now has zero diff vs main. The reverted ledger changes are gone and the branch is clean.";
3243        let ndjson = format!(
3244            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{prose:?},"usage":{{"input_tokens":2500,"output_tokens":300}}}}"#
3245        );
3246
3247        let runner = agents::get_runner("claude");
3248        let parsed = parse_success_output("149164", "claude", &*runner, &ndjson)
3249            .expect("should synthesize done from NDJSON envelope when result is plain prose");
3250
3251        assert_eq!(parsed.response.status, "done");
3252        assert_eq!(parsed.response.summary, prose);
3253        assert_eq!(parsed.input_tokens, Some(2500));
3254        assert_eq!(parsed.output_tokens, Some(300));
3255    }
3256
3257    /// Verify that is_error=true in NDJSON envelope still returns AgentFailed,
3258    /// not "done" — the envelope fallback must not override explicit errors.
3259    #[test]
3260    fn parse_success_output_ndjson_envelope_fallback_skipped_when_is_error_true() {
3261        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"result":"Something went wrong with no status"}"#;
3262
3263        let runner = agents::get_runner("claude");
3264        let err = parse_success_output("149999", "claude", &*runner, ndjson)
3265            .expect_err("is_error=true should return AgentFailed");
3266
3267        assert!(matches!(err, agents::AgentError::AgentFailed { .. }));
3268    }
3269
3270    fn tool_pair(id: &str, name: &str, cmd: &str, is_error: bool) -> String {
3271        format!(
3272            "{}\n{}\n",
3273            serde_json::json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":id,"name":name,"input":{"command":cmd}}]}}),
3274            serde_json::json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":id,"is_error":is_error,"content":"x"}]}}),
3275        )
3276    }
3277
3278    #[test]
3279    fn salvage_wrapup_accepts_commit_then_cleanup() {
3280        let raw = tool_pair("1", "run_terminal_cmd", "git commit -m x", false)
3281            + &tool_pair("2", "run_terminal_cmd", "git status", false);
3282        assert!(salvage_wrapup(&raw).is_some());
3283    }
3284
3285    #[test]
3286    fn salvage_wrapup_rejects_edit_after_commit_or_unanswered_call() {
3287        let edit = tool_pair("1", "run_terminal_cmd", "git commit -m x", false)
3288            + &tool_pair("2", "search_replace", "", false);
3289        assert!(salvage_wrapup(&edit).is_none());
3290        let pending = tool_pair("1", "run_terminal_cmd", "git commit -m x", false)
3291            + &serde_json::json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":"2","name":"run_terminal_cmd","input":{"command":"cargo test"}}]}}).to_string();
3292        assert!(salvage_wrapup(&pending).is_none());
3293        assert!(salvage_wrapup(&tool_pair("1", "run_terminal_cmd", "ls", false)).is_none());
3294    }
3295
3296    // ── terminal_reason:completed + exit 1 detection (issue #3087) ─────────────
3297
3298    /// kimi/minimax sometimes emit valid NDJSON with terminal_reason=completed
3299    /// but exit 1. In that case parse_success_output may fail (e.g. empty result
3300    /// text that synthesize can't parse), then parse_session_output would
3301    /// normally fall back to classify_error — grabbing a garbled JSON tail and
3302    /// producing a spurious Unknown/Auth error. The fix intercepts this and
3303    /// returns InvalidResponse (soft error, re-route) instead.
3304    #[test]
3305    fn parse_session_output_exit0_without_envelope_is_interrupted() {
3306        let session = session::SessionOutput {
3307            exit_code: 0,
3308            raw_stdout: "\u{4}\u{4}\u{4} truncated pane junk".to_string(),
3309            raw_stderr: String::new(),
3310            elapsed_secs: Some(180),
3311            sandbox_error: None,
3312        };
3313
3314        let runner = agents::get_runner("claude");
3315        let err = parse_session_output("3876", "claude", &*runner, &session)
3316            .expect_err("missing envelope should be an error");
3317        assert!(
3318            matches!(err, agents::AgentError::Interrupted { .. }),
3319            "exit 0 with no result envelope should be Interrupted, got {err:?}"
3320        );
3321    }
3322
3323    #[test]
3324    fn parse_session_output_exit1_with_terminal_reason_completed_returns_invalid_response() {
3325        // Empty result text causes synthesize_response_from_text to return None
3326        // (empty/whitespace-only input), so parse_success_output returns
3327        // InvalidResponse. With the fix, parse_session_output intercepts the
3328        // "terminal_reason":"completed" signal before classify_error and
3329        // returns InvalidResponse (re-route) instead of a garbled error.
3330        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":"","usage":{"input_tokens":1000,"output_tokens":100}}"#;
3331        let session = session::SessionOutput {
3332            exit_code: 1,
3333            raw_stdout: ndjson.to_string(),
3334            raw_stderr: String::new(),
3335            elapsed_secs: Some(30),
3336            sandbox_error: None,
3337        };
3338
3339        let runner = agents::get_runner("claude");
3340        let result = parse_session_output("149298", "claude", &*runner, &session);
3341
3342        let err = result.expect_err("should be an error, not Ok");
3343        assert!(
3344            matches!(err, agents::AgentError::InvalidResponse { .. }),
3345            "should be InvalidResponse, got {err:?}"
3346        );
3347    }
3348
3349    // ── GLM cost-telemetry exit-1 detection (issue #3094) ──────────────────────
3350
3351    /// GLM exits with code 1 but stdout is only cost telemetry JSON (contains
3352    /// "costUSD":). This must return InvalidResponse (soft re-route) instead of
3353    /// a hard Failed classification with spurious agent cooldown.
3354    #[test]
3355    fn parse_session_output_exit1_with_cost_telemetry_returns_invalid_response() {
3356        // GLM emits cost/usage JSON on clean exit; exit code 1 is a false failure.
3357        let telemetry = r#"{"type":"result","subtype":"success","is_error":false,"result":"","costUSD":0.042,"usage":{"input_tokens":5000,"output_tokens":800}}"#;
3358        let session = session::SessionOutput {
3359            exit_code: 1,
3360            raw_stdout: telemetry.to_string(),
3361            raw_stderr: String::new(),
3362            elapsed_secs: Some(120),
3363            sandbox_error: None,
3364        };
3365
3366        let runner = agents::get_runner("glm");
3367        let result = parse_session_output("149347", "glm", &*runner, &session);
3368
3369        let err = result.expect_err("should be an error, not Ok");
3370        assert!(
3371            matches!(err, agents::AgentError::InvalidResponse { .. }),
3372            "glm cost-telemetry exit-1 should return InvalidResponse, got {err:?}"
3373        );
3374    }
3375
3376    /// GLM with costUSD AND is_error:true — must propagate AgentFailed, not
3377    /// silently swallow the error via the cost-telemetry guard.
3378    #[test]
3379    fn parse_session_output_exit1_cost_telemetry_with_is_error_true_propagates_failure() {
3380        let telemetry = r#"{"type":"result","subtype":"error","is_error":true,"result":"tool failed","costUSD":0.01,"usage":{"input_tokens":100,"output_tokens":10}}"#;
3381        let session = session::SessionOutput {
3382            exit_code: 1,
3383            raw_stdout: telemetry.to_string(),
3384            raw_stderr: String::new(),
3385            elapsed_secs: Some(15),
3386            sandbox_error: None,
3387        };
3388
3389        let runner = agents::get_runner("glm");
3390        let result = parse_session_output("149348", "glm", &*runner, &session);
3391
3392        let err = result.expect_err("should be an error");
3393        assert!(
3394            matches!(err, agents::AgentError::AgentFailed { .. }),
3395            "is_error=true with costUSD must still return AgentFailed, got {err:?}"
3396        );
3397    }
3398
3399    /// terminal_reason=completed must NOT override actual error conditions.
3400    /// When is_error=true in the NDJSON envelope, AgentFailed must be returned
3401    /// even if terminal_reason=completed is also present. Only kimi/minimax
3402    /// agents use find_claude_result which extracts is_error, so use "kimi"
3403    /// as agent_name.
3404    #[test]
3405    fn parse_session_output_exit1_preserves_error_when_is_error_true() {
3406        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"terminal_reason":"completed","result":"failed"}"#;
3407        let session = session::SessionOutput {
3408            exit_code: 1,
3409            raw_stdout: ndjson.to_string(),
3410            raw_stderr: String::new(),
3411            elapsed_secs: Some(30),
3412            sandbox_error: None,
3413        };
3414
3415        let runner = agents::get_runner("kimi");
3416        let result = parse_session_output("149299", "kimi", &*runner, &session);
3417
3418        let err = result.expect_err("should be an error");
3419        assert!(
3420            matches!(err, agents::AgentError::AgentFailed { .. }),
3421            "is_error=true should return AgentFailed, got {err:?}"
3422        );
3423    }
3424
3425    /// RateLimit with terminal_reason=completed must be propagated as RateLimit,
3426    /// not downgraded to InvalidResponse. This covers claude session-limit errors
3427    /// that include "terminal_reason":"completed" in their NDJSON envelope — the
3428    /// terminal_reason guard was designed for kimi/minimax/glm false-positive
3429    /// exit codes but was intercepting classified errors like RateLimit.
3430    #[test]
3431    fn parse_session_output_exit1_with_rate_limit_and_terminal_reason_completed() {
3432        let ndjson = r#"{"type":"result","is_error":true,"terminal_reason":"completed","result":"You've hit your session limit · resets 2:20pm (America/Sao_Paulo)","usage":{"input_tokens":0,"output_tokens":0}}"#;
3433        let session = session::SessionOutput {
3434            exit_code: 1,
3435            raw_stdout: ndjson.to_string(),
3436            raw_stderr: String::new(),
3437            elapsed_secs: Some(120),
3438            sandbox_error: None,
3439        };
3440
3441        let runner = agents::get_runner("claude");
3442        let result = parse_session_output("152895", "claude", &*runner, &session);
3443
3444        let err = result.expect_err("rate-limit should be an error");
3445        assert!(
3446            matches!(err, agents::AgentError::RateLimit { .. }),
3447            "session limit should return RateLimit, not downgraded, got {err:?}"
3448        );
3449    }
3450}