Skip to main content

orch/engine/runner/
mod.rs

1//! Task runner — executes tasks using AI agents.
2//!
3//! Replaces `run_task.sh` with pure Rust. The runner:
4//! 1. Sets up a git worktree for isolation
5//! 2. Builds context (project instructions, skills, comments)
6//! 3. Renders prompts and spawns the agent in tmux
7//! 4. Waits for completion, collects and classifies the response
8//! 5. Handles errors (reroute on limits, fallback agents)
9//! 6. Auto-commits, pushes, and creates PRs
10//! 7. Updates status labels and posts result comments
11//!
12//! Submodules handle the heavy lifting:
13//! - [`task_init`] — guard checks, worktree setup, and invocation building
14//! - [`session`] — tmux session lifecycle and output collection
15//! - [`response_handler`] — success path: commit, push, PR, token storage
16//! - [`fallback`] — error classification and recovery strategies
17
18pub mod agent;
19pub mod agents;
20pub mod context;
21pub mod cow;
22pub mod diff;
23pub mod direct;
24pub mod fallback;
25pub mod git_ops;
26pub mod response;
27pub mod response_handler;
28pub mod session;
29pub mod task_init;
30pub mod worktree;
31
32use crate::backends::{ExternalBackend, ExternalTask, Status};
33use crate::config;
34use crate::engine::router::RouteResult;
35use crate::engine::tasks::is_internal_id;
36use crate::security;
37use crate::store;
38use crate::store::InsertTaskMetric;
39use crate::tmux::TmuxManager;
40use anyhow::Context;
41use chrono::Utc;
42pub use response::WeightSignal;
43use std::path::PathBuf;
44use std::sync::Arc;
45
46/// Fully materialized task-run audit data.
47#[derive(Debug, Clone)]
48pub struct RunAudit {
49    pub stdout: String,
50    pub stderr: String,
51    pub parsed_response: String,
52    pub outcome: String,
53    pub error: String,
54    pub input_tokens: u64,
55    pub output_tokens: u64,
56    pub total_cost_usd: f64,
57    pub duration_secs: f64,
58}
59
60/// Result returned by `run()` so callers can persist the audit trail.
61#[derive(Debug, Clone)]
62pub struct RunExecution {
63    pub status: String,
64    pub exit_code: Option<i32>,
65    pub audit: RunAudit,
66}
67
68fn serialize_parsed_response(
69    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
70    raw_stdout: &str,
71) -> String {
72    match parse_result {
73        Ok(parsed) => serde_json::json!({
74            "response": parsed.response,
75            "input_tokens": parsed.input_tokens,
76            "output_tokens": parsed.output_tokens,
77            "duration_ms": parsed.duration_ms,
78        })
79        .to_string(),
80        Err(agents::AgentError::InvalidResponse { raw }) => {
81            serde_json::json!({"raw": raw}).to_string()
82        }
83        Err(_) => serde_json::json!({"raw": raw_stdout}).to_string(),
84    }
85}
86
87fn classify_run_outcome(
88    status: &str,
89    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
90    push_failed: bool,
91) -> &'static str {
92    if push_failed {
93        return "push_failed";
94    }
95    match parse_result {
96        Err(agents::AgentError::Timeout { .. }) => "timeout",
97        // Generic rate-limit patterns (e.g. "usage limit") also match billing-cycle
98        // exhaustion messages, so consult the credit-exhaustion detector first —
99        // otherwise Kimi-style "usage limit for this billing cycle" errors are
100        // stored as a plain "rate_limit" outcome (issue #3529).
101        Err(agents::AgentError::RateLimit { message }) => {
102            crate::engine::cooldown::detect_credit_exhaustion_outcome(message)
103                .unwrap_or("rate_limit")
104        }
105        Err(agents::AgentError::Auth { message }) => {
106            crate::engine::cooldown::detect_credit_exhaustion_outcome(message).unwrap_or("failed")
107        }
108        Err(agents::AgentError::InvalidResponse { .. }) => "parse_error",
109        Err(_) => "failed",
110        Ok(_)
111            if matches!(
112                status,
113                "done" | "completed" | "in_progress" | "in_review" | "needs_review" | "routed"
114            ) =>
115        {
116            "success"
117        }
118        Ok(_) if status == "blocked" => "blocked",
119        // Non-canonical status that clearly reads as completion — success, not
120        // "unrecognized status" (#3627).
121        Ok(_) if crate::status_heuristics::status_looks_like_descriptive_completion(status) => {
122            "success"
123        }
124        Ok(_) => "failed",
125    }
126}
127
128fn classify_run_error_type(last_error: &str) -> &'static str {
129    if last_error.is_empty() {
130        // No error: agent completed successfully and created a PR waiting for review
131        "success"
132    } else if last_error.contains("timeout") {
133        "timeout"
134    } else if last_error.contains("billing cycle") || last_error.contains("monthly quota") {
135        "billing_cycle_exhausted"
136    } else if last_error.contains("rate limit") || last_error.contains("usage limit") {
137        "rate_limit"
138    } else if last_error.contains("auth") {
139        "auth_error"
140    } else if last_error.contains("push failed") {
141        "push_failed"
142    } else if last_error.contains("No commits between") || last_error.contains("no commits") {
143        "no_commits"
144    } else if last_error.contains("create PR failed")
145        || last_error.contains("failed to create pull request")
146        || last_error.contains("pull request creation failed")
147    {
148        "pr_failed"
149    } else if last_error.contains("invalid response") || last_error.contains("parse error") {
150        "parse_error"
151    } else if last_error.contains("exceeded max attempts") {
152        "max_attempts"
153    } else {
154        let lower = last_error.to_ascii_lowercase();
155        if lower.contains("build failed")
156            || lower.contains("tests failed")
157            || lower.contains("test failed")
158            || lower.contains("test suite failed")
159            || lower.contains("lint failed")
160        {
161            "ci_failure"
162        } else {
163            "failed"
164        }
165    }
166}
167
168fn extract_run_tokens(
169    parse_result: &Result<agents::ParsedResponse, agents::AgentError>,
170) -> (u64, u64) {
171    match parse_result {
172        Ok(parsed) => (
173            parsed
174                .input_tokens
175                .or(parsed.response.input_tokens)
176                .unwrap_or(0),
177            parsed
178                .output_tokens
179                .or(parsed.response.output_tokens)
180                .unwrap_or(0),
181        ),
182        Err(_) => (0, 0),
183    }
184}
185
186fn run_duration_secs(started_at: &chrono::DateTime<Utc>) -> f64 {
187    (Utc::now() - *started_at).num_milliseconds() as f64 / 1000.0
188}
189
190struct RunAuditInput<'a> {
191    task_id: &'a str,
192    status: &'a str,
193    parse_result: &'a Result<agents::ParsedResponse, agents::AgentError>,
194    raw_stdout: &'a str,
195    raw_stderr: &'a str,
196    started_at: &'a chrono::DateTime<Utc>,
197    error_override: Option<String>,
198    elapsed_secs: Option<u64>,
199    push_failed: bool,
200}
201
202fn parse_success_output(
203    task_id: &str,
204    agent_name: &str,
205    agent_runner: &dyn agents::AgentRunner,
206    raw_stdout: &str,
207) -> Result<agents::ParsedResponse, agents::AgentError> {
208    // Quick NDJSON terminal_reason:completed shortcut: when the agent
209    // emitted a session envelope that explicitly signals completion we
210    // prefer to interpret that as a successful run (or at least a
211    // soft/empty success) rather than immediately classifying an error
212    // from the trailing process output.  Do not override explicit
213    // in-envelope `is_error: true` signals.
214    if raw_stdout.contains("\"terminal_reason\":\"completed\"")
215        && !raw_stdout.contains("\"is_error\":true")
216    {
217        // Try to recover a structured response from assistant messages first
218        let assistant_text = agents::collect_assistant_messages_text(agent_name, raw_stdout);
219        if !assistant_text.is_empty() {
220            if let Ok(response) = crate::parser::parse(&assistant_text) {
221                tracing::debug!(task_id, agent = agent_name, "found AgentResponse JSON in earlier assistant message (terminal_reason:completed)");
222                return Ok(agents::ParsedResponse {
223                    response,
224                    input_tokens: None,
225                    output_tokens: None,
226                    duration_ms: None,
227                });
228            }
229            if let Some(response) = agents::synthesize_response_from_text(&assistant_text) {
230                tracing::warn!(
231                    task_id,
232                    agent = agent_name,
233                    "synthesizing response from assistant message due to terminal_reason:completed"
234                );
235                return Ok(agents::ParsedResponse {
236                    response,
237                    input_tokens: None,
238                    output_tokens: None,
239                    duration_ms: None,
240                });
241            }
242        }
243        // If we couldn't recover structured data from earlier assistant
244        // messages, fall through and let the regular NDJSON envelope parsing
245        // / agent-specific parsers handle the terminal_reason case. This
246        // preserves the existing behavior where an empty `result` field
247        // combined with exit != 0 becomes an InvalidResponse that is then
248        // interpreted by parse_session_output (see tests).
249    }
250
251    // 1. Try to use the unified agent result extractor first
252    if let Some(agent_result) = agents::find_agent_result(agent_name, raw_stdout) {
253        // Treat is_error as authoritative for the success path
254        if agent_result.is_error {
255            // Classify the error from the agent's result text rather than
256            // blindly returning AgentFailed — this preserves rate-limit, auth,
257            // and other actionable error types for proper cooldown/reroute.
258            // Without this, e.g. a claude session-limit error with is_error=true
259            // would be downgraded to AgentFailed, losing the RateLimit signal
260            // that the fallback chain needs for UsageLimit recovery (issue #3292).
261            if let Some(e) = agents::patterns::detect_rate_limit(&agent_result.result_text) {
262                return Err(e);
263            }
264            if let Some(e) = agents::patterns::detect_auth_error(&agent_result.result_text) {
265                return Err(e);
266            }
267            if let Some(e) = agents::patterns::detect_model_unavailable(&agent_result.result_text) {
268                return Err(e);
269            }
270            return Err(agents::AgentError::AgentFailed {
271                message: agent_result.result_text,
272            });
273        }
274
275        // Try to parse the inner result text as standard JSON
276        match crate::parser::parse(&agent_result.result_text) {
277            Ok(response) => {
278                return Ok(agents::ParsedResponse {
279                    response,
280                    input_tokens: agent_result.input_tokens,
281                    output_tokens: agent_result.output_tokens,
282                    duration_ms: agent_result.duration_ms,
283                });
284            }
285            Err(_) => {
286                // Strategy 2: scan all earlier assistant messages in the NDJSON
287                // stream for valid AgentResponse JSON. Agents often emit the
288                // JSON in an intermediate turn and then summarise in prose as
289                // their final output, so the `type:result` text may be pure
290                // narrative while a prior assistant turn contains the JSON.
291                let assistant_text =
292                    agents::collect_assistant_messages_text(agent_name, raw_stdout);
293                if !assistant_text.is_empty() {
294                    if let Ok(response) = crate::parser::parse(&assistant_text) {
295                        tracing::debug!(
296                            task_id,
297                            agent = agent_name,
298                            "found AgentResponse JSON in earlier assistant message"
299                        );
300                        return Ok(agents::ParsedResponse {
301                            response,
302                            input_tokens: agent_result.input_tokens,
303                            output_tokens: agent_result.output_tokens,
304                            duration_ms: agent_result.duration_ms,
305                        });
306                    }
307                }
308
309                // Only fall back to text synthesis when parsing fails
310                if let Some(response) =
311                    agents::synthesize_response_from_text(&agent_result.result_text)
312                {
313                    tracing::warn!(
314                        task_id,
315                        agent = agent_name,
316                        "parse failed on agent result, synthesizing response from plain text"
317                    );
318                    return Ok(agents::ParsedResponse {
319                        response,
320                        input_tokens: agent_result.input_tokens,
321                        output_tokens: agent_result.output_tokens,
322                        duration_ms: agent_result.duration_ms,
323                    });
324                }
325
326                // NDJSON envelope fallback: the envelope explicitly reports
327                // success (is_error=false) but the result content is domain
328                // JSON or plain prose that doesn't match AgentResponse schema
329                // and doesn't contain completion keywords.  Rather than
330                // returning InvalidResponse (which causes the task to re-run),
331                // synthesize a minimal "done" response from the result text.
332                //
333                // Guard against kimi/minimax edge case: when terminal_reason=completed
334                // but result text is empty, synthesize returns None, parse_success_output
335                // returns InvalidResponse, and parse_session_output can intercept it
336                // before classify_error. Requiring non-empty result here means the
337                // envelope fallback only fires when we have actual text to report.
338                if !agent_result.is_error
339                    && raw_stdout.contains("\"terminal_reason\":\"completed\"")
340                    && !agent_result.result_text.is_empty()
341                {
342                    let summary_end =
343                        agents::truncate_at_char_boundary(&agent_result.result_text, 500);
344                    let summary = agent_result.result_text[..summary_end].to_string();
345                    tracing::warn!(
346                        task_id,
347                        agent = agent_name,
348                        "NDJSON envelope indicates success but result lacks AgentResponse \
349                         schema — synthesizing done response"
350                    );
351                    return Ok(agents::ParsedResponse {
352                        response: crate::parser::AgentResponse {
353                            status: "done".to_string(),
354                            summary,
355                            ..Default::default()
356                        },
357                        input_tokens: agent_result.input_tokens,
358                        output_tokens: agent_result.output_tokens,
359                        duration_ms: agent_result.duration_ms,
360                    });
361                }
362            }
363        }
364    }
365
366    // 2. Fall back to generic agent-specific parser if extraction failed
367    match agent_runner.parse_response(raw_stdout) {
368        Ok(parsed) => Ok(parsed),
369        Err(agents::AgentError::InvalidResponse { raw }) => {
370            if let Some(response) = agents::synthesize_response_from_text(&raw) {
371                tracing::warn!(
372                    task_id,
373                    agent = agent_name,
374                    "parse failed, synthesizing response from plain text"
375                );
376                // Try to extract token metadata from the original raw NDJSON
377                // output (not the truncated/extracted text from the error), so
378                // cost tracking isn't lost for agents that return free-form text.
379                let agent_result = agents::find_agent_result(agent_name, raw_stdout);
380                let input_tokens = agent_result.as_ref().and_then(|r| r.input_tokens);
381                let output_tokens = agent_result.as_ref().and_then(|r| r.output_tokens);
382                let duration_ms = agent_result.as_ref().and_then(|r| r.duration_ms);
383                Ok(agents::ParsedResponse {
384                    response,
385                    input_tokens,
386                    output_tokens,
387                    duration_ms,
388                })
389            } else {
390                Err(agents::AgentError::InvalidResponse { raw })
391            }
392        }
393        Err(err) => Err(err),
394    }
395}
396
397/// Parse session output, handling the kimi/minimax/glm exit-1-with-completed-edge case.
398///
399/// Tries `parse_success_output` first when stdout is non-empty. If that succeeds,
400/// returns success regardless of exit code. If it fails and `exit_code != 0`,
401/// checks for `"terminal_reason":"completed"` or cost-telemetry-only stdout
402/// (`"costUSD":`) in stdout before falling back to `classify_error` — so valid
403/// sessions that exit 1 produce `InvalidResponse` (soft error, re-route) instead
404/// of a garbled `Unknown`/`Auth` error (hard failure with spurious cooldown).
405pub fn parse_session_output(
406    task_id: &str,
407    agent_name: &str,
408    agent_runner: &dyn agents::AgentRunner,
409    session_output: &session::SessionOutput,
410) -> Result<agents::ParsedResponse, agents::AgentError> {
411    // Some agents may emit NDJSON/telemetry to stderr instead of stdout. Combine
412    // both streams so parse_success_output can inspect either place for the
413    // terminal_reason/cost telemetry envelope. Only skip parsing when both are
414    // empty so we still attempt recovery when stderr contains the envelope.
415    let combined_output = format!("{}{}", session_output.raw_stdout, session_output.raw_stderr);
416    if !(session_output.raw_stdout.is_empty() && session_output.raw_stderr.is_empty()) {
417        match parse_success_output(task_id, agent_name, agent_runner, &combined_output) {
418            Ok(parsed) => return Ok(parsed),
419            Err(err) => {
420                // If parse_success_output already returned a classified error
421                // (RateLimit, AgentFailed, Timeout, Auth, etc.), propagate it
422                // directly — only InvalidResponse (unparseable output) should
423                // be intercepted by the terminal_reason:completed / cost-telemetry
424                // guard below. Without this, e.g. a claude session-limit error
425                // with terminal_reason=completed would be downgraded from RateLimit
426                // to InvalidResponse → Failed, losing the correct cooldown/reroute.
427                if !matches!(err, agents::AgentError::InvalidResponse { .. }) {
428                    return Err(err);
429                }
430                if session_output.exit_code != 0 {
431                    // kimi/minimax/glm sometimes emit valid NDJSON with
432                    // "terminal_reason":"completed" or cost-only telemetry
433                    // ("costUSD":) but exit 1 due to a post-session hook or
434                    // minor CLI issue. Scanning the tail would just grab JSON
435                    // metadata, producing a garbled error. Detect either
436                    // completion signal and treat as a soft parse error so the
437                    // task re-routes instead of hitting a false cooldown from
438                    // Unknown or Auth error variants. Use the combined output
439                    // (stdout+stderr) so envelopes written to stderr are
440                    // observed.
441                    let combined = &combined_output;
442                    let is_completed_signal =
443                        combined.contains("\"terminal_reason\":\"completed\"");
444                    // GLM emits a cost-telemetry JSON object (containing
445                    // "costUSD":) as its final stdout/stderr line on clean
446                    // exit even when the process exits with code 1. If the
447                    // only content is such telemetry (no is_error=true in the
448                    // envelope), treat it the same as terminal_reason:completed.
449                    let is_cost_telemetry_only = combined.contains("\"costUSD\":")
450                        && !combined.contains("\"is_error\":true");
451                    if is_completed_signal || is_cost_telemetry_only {
452                        let raw_tail = agents::patterns::safe_tail(combined, 300);
453                        return Err(agents::AgentError::InvalidResponse {
454                            raw: raw_tail.to_string(),
455                        });
456                    }
457                    return Err(agent_runner.classify_error_with_elapsed(
458                        session_output.exit_code,
459                        &session_output.raw_stdout,
460                        &session_output.raw_stderr,
461                        session_output.elapsed_secs,
462                    ));
463                }
464                return Err(err);
465            }
466        }
467    }
468
469    if session_output.exit_code != 0 {
470        return Err(agent_runner.classify_error_with_elapsed(
471            session_output.exit_code,
472            &session_output.raw_stdout,
473            &session_output.raw_stderr,
474            session_output.elapsed_secs,
475        ));
476    }
477
478    Err(agents::AgentError::Unknown {
479        exit_code: session_output.exit_code,
480        message: "empty-output-exit0: opencode returned exit 0 with empty stdout".to_string(),
481    })
482}
483
484/// Task runner configuration.
485pub struct TaskRunner {
486    /// Repository slug (owner/repo)
487    repo: String,
488    /// Path to the orch home directory
489    orch_home: PathBuf,
490    /// Unified task store for run audit trail
491    store: Option<Arc<crate::store::TaskStore>>,
492}
493
494impl TaskRunner {
495    pub fn new(repo: String) -> Self {
496        let orch_home = crate::home::orch_home().unwrap_or_else(|e| {
497            tracing::warn!(error = %e, "orch_home() failed — falling back to /tmp/.orch");
498            PathBuf::from("/tmp").join(".orch")
499        });
500
501        Self {
502            repo,
503            orch_home,
504            store: None,
505        }
506    }
507
508    /// Set the unified task store for run audit trail.
509    pub fn with_store(mut self, store: Arc<crate::store::TaskStore>) -> Self {
510        self.store = Some(store);
511        self
512    }
513
514    /// Read a field from the task store.
515    async fn get_field(&self, task_id: &str, field: &str) -> Option<String> {
516        let task = store::opt_store_get_task(&self.store, &self.repo, task_id).await?;
517        match field {
518            "route_reason" => Some(task.route_reason),
519            "worktree" => Some(task.worktree),
520            "attempts" => Some(task.attempts.to_string()),
521            "delegations" => serde_json::to_string(&task.delegations).ok(),
522            "summary" => Some(task.summary),
523            "last_error" => Some(task.last_error),
524            "agent" => task.agent,
525            _ => None,
526        }
527    }
528
529    async fn build_run_audit(&self, input: RunAuditInput<'_>) -> RunAudit {
530        // Filter empty strings so that a blank last_error does not shadow the
531        // actual parse_result error.  Without this, `Option::or(Some(""))` would
532        // return `Some("")` and prevent the `unwrap_or_else` fallback from ever
533        // running, causing task_runs.error to be written as "" even when the
534        // agent returned a classifiable error (fixes issue #2527).
535        let last_error = self
536            .get_field(input.task_id, "last_error")
537            .await
538            .filter(|s| !s.is_empty());
539        let outcome =
540            classify_run_outcome(input.status, input.parse_result, input.push_failed).to_string();
541
542        let error = if outcome == "success" {
543            String::new()
544        } else {
545            let error = match input.error_override.as_deref().filter(|s| !s.is_empty()) {
546                Some(error_override) => error_override.to_string(),
547                None => match last_error {
548                    Some(last_error) => last_error,
549                    None => match input.parse_result {
550                        Ok(parsed) => parsed.response.error.clone().unwrap_or_default(),
551                        Err(err) => err.to_string(),
552                    },
553                },
554            };
555
556            if error.is_empty() {
557                if input.status == "blocked" {
558                    match input.parse_result {
559                        Ok(parsed) if !parsed.response.summary.is_empty() => {
560                            parsed.response.summary.clone()
561                        }
562                        _ => "task blocked without explicit error".to_string(),
563                    }
564                } else if matches!(
565                    input.status,
566                    "done"
567                        | "completed"
568                        | "in_progress"
569                        | "in_review"
570                        | "needs_review"
571                        | "routed"
572                        | "new"
573                ) {
574                    // Canonical non-success statuses have no error message (e.g. "routed").
575                    String::new()
576                } else {
577                    // Non-canonical status with no error message — surface the status
578                    // explicitly so operators can identify which statuses need to be
579                    // added to the normalization map in parser.rs.
580                    format!("unrecognized status: {}", input.status)
581                }
582            } else {
583                error
584            }
585        };
586
587        let total_cost_usd = match &self.store {
588            Some(_) => {
589                store::get_cost_estimate(&self.store, &self.repo, input.task_id)
590                    .await
591                    .total_cost_usd
592            }
593            None => 0.0,
594        };
595
596        let (input_tokens, output_tokens) = extract_run_tokens(input.parse_result);
597
598        let duration_secs = input
599            .elapsed_secs
600            .map(|s| s as f64)
601            .unwrap_or_else(|| run_duration_secs(input.started_at));
602
603        RunAudit {
604            stdout: input.raw_stdout.to_string(),
605            stderr: input.raw_stderr.to_string(),
606            parsed_response: serialize_parsed_response(input.parse_result, input.raw_stdout),
607            outcome,
608            error,
609            input_tokens,
610            output_tokens,
611            total_cost_usd,
612            duration_secs,
613        }
614    }
615
616    /// Run a task through the full execution pipeline.
617    ///
618    /// Returns `Ok(None)` if the runner guard skipped the task (caller should not
619    /// post stale results). Returns `Ok(Some(status))` with the final task status.
620    pub async fn run(
621        &self,
622        task_id: &str,
623        agent: Option<&str>,
624        model: Option<&str>,
625        backend: Option<&dyn ExternalBackend>,
626        started_at: &chrono::DateTime<Utc>,
627    ) -> anyhow::Result<Option<RunExecution>> {
628        tracing::info!(
629            task_id,
630            agent = agent.unwrap_or("default"),
631            model = model.unwrap_or("default"),
632            "starting task execution"
633        );
634
635        // Check task guards; returns outcome indicating whether to proceed.
636        let attempts = match task_init::check_guards(task_id, &self.repo, &self.store).await {
637            Ok(task_init::GuardOutcome::Proceed(a)) => a,
638            Ok(task_init::GuardOutcome::Skip) => {
639                return Ok(None);
640            }
641            Ok(task_init::GuardOutcome::MaxAttempts) => {
642                // The caller marks the task blocked from the returned status, a task that ran out
643                // of attempts did not finish, so it must never reach done
644                if backend.is_some() {
645                    // If this was label-forced to a specific agent, remove the agent label
646                    // so that /retry can route to a different agent instead of looping.
647                    let route_reason = self
648                        .get_field(task_id, "route_reason")
649                        .await
650                        .unwrap_or_default();
651                    if route_reason.starts_with("label agent:") {
652                        let agent_label = route_reason.trim_start_matches("label ");
653                        let gh = crate::github::http::GhHttp::new()?;
654                        if let Err(e) = gh.remove_label(&self.repo, task_id, agent_label).await {
655                            tracing::warn!(task_id, label = agent_label, err = %e, "failed to remove agent label after max attempts");
656                        } else {
657                            tracing::info!(task_id, label = agent_label, "removed forced agent label after max attempts — /retry will use free routing");
658                        }
659                    }
660                }
661                return Ok(Some(RunExecution {
662                    status: "blocked".to_string(),
663                    exit_code: None,
664                    audit: RunAudit {
665                        stdout: String::new(),
666                        stderr: String::new(),
667                        parsed_response: String::new(),
668                        outcome: "failed".to_string(),
669                        error: "max attempts reached".to_string(),
670                        input_tokens: 0,
671                        output_tokens: 0,
672                        total_cost_usd: 0.0,
673                        duration_secs: 0.0,
674                    },
675                }));
676            }
677            Err(e) => return Err(e),
678        };
679
680        // Resolve project directory
681        let project_dir = self.resolve_project_dir().await?;
682
683        // Prepare task: worktree, context, prompts, and agent invocation
684        let init = task_init::prepare_task(
685            task_id,
686            agent,
687            model,
688            backend,
689            &self.repo,
690            &project_dir,
691            attempts,
692            &self.store,
693        )
694        .await?;
695
696        // Run agent session: spawn in tmux, wait for completion, collect output
697        let (tmux, tmux_session, session_output) = session::run_agent_session(
698            task_id,
699            &init.invocation,
700            &init.attempt_dir,
701            &self.orch_home,
702        )
703        .await;
704
705        // Touch updated_at immediately after session exit so the stuck-task recovery
706        // timer resets before response_handler post-processing (git push, PR creation,
707        // status update) begins.  Without this, a session that ran longer than
708        // no_session_stuck_timeout (default 10 min) would be re-dispatched by the tick
709        // while the runner is still writing results.
710        store::store_touch_updated_at(&self.store, &self.repo, task_id).await;
711
712        // If silence detection (tick phase1b) already rerouted this task while the
713        // session was running, skip all fallback/review processing. Without this check the
714        // runner would call `fallback::handle_error` on the empty output, exhaust fallback
715        // agents, and mark the task `needs_review` — overwriting the rerouted status and
716        // triggering a spurious review cycle.
717        // Silence detection now sets Routed (with fallback agent) or NeedsReview (no fallback).
718        // Check for both, plus New for backward compatibility.
719        if let Some(stored) = store::opt_store_get_task(&self.store, &self.repo, task_id).await {
720            let silence_reset = matches!(
721                stored.status,
722                crate::store::TaskStatus::New
723                    | crate::store::TaskStatus::Routed
724                    | crate::store::TaskStatus::NeedsReview
725            );
726            if silence_reset {
727                let status_str = stored.status.as_str();
728                tracing::info!(
729                    task_id,
730                    status = status_str,
731                    "task already reset by silence detection — skipping fallback processing"
732                );
733                session::cleanup_session(task_id, &tmux, &tmux_session).await;
734                let silence_err = agents::patterns::classify_from_text(0, "silence detected");
735                let silence_parse: Result<agents::ParsedResponse, agents::AgentError> =
736                    Err(silence_err);
737                let audit = self
738                    .build_run_audit(RunAuditInput {
739                        task_id,
740                        status: status_str,
741                        parse_result: &silence_parse,
742                        raw_stdout: &session_output.raw_stdout,
743                        raw_stderr: &session_output.raw_stderr,
744                        started_at,
745                        error_override: Some(format!("silence detection set task to {status_str}")),
746                        elapsed_secs: session_output.elapsed_secs,
747                        push_failed: false,
748                    })
749                    .await;
750                return Ok(Some(RunExecution {
751                    status: status_str.to_string(),
752                    exit_code: Some(session_output.exit_code),
753                    audit,
754                }));
755            }
756        }
757
758        // Log raw output for debugging agent failures
759        let stdout_len = session_output.raw_stdout.len();
760        let stderr_len = session_output.raw_stderr.len();
761        let stdout_tail: String = session_output
762            .raw_stdout
763            .chars()
764            .rev()
765            .take(500)
766            .collect::<String>()
767            .chars()
768            .rev()
769            .collect();
770        let stderr_tail: String = session_output
771            .raw_stderr
772            .chars()
773            .rev()
774            .take(500)
775            .collect::<String>()
776            .chars()
777            .rev()
778            .collect();
779        tracing::info!(
780            task_id,
781            exit_code = session_output.exit_code,
782            stdout_len,
783            stderr_len,
784            stdout_tail = %stdout_tail,
785            stderr_tail = %stderr_tail,
786            "agent raw output"
787        );
788
789        // Use agent-specific parsing where possible. Previously we only attempted
790        // parse_success_output when exit_code == 0 which caused runs that emitted
791        // valid NDJSON but exited non-zero (kimi / minimax) to be classed as
792        // failures. Try parsing success output first when stdout is present; if
793        // parsing succeeds treat as success regardless of exit code. Only fall
794        // back to error classification when parsing fails and the process exit
795        // code indicates an error.
796        let agent_runner = agents::get_runner(&init.agent_name);
797        let parse_result =
798            parse_session_output(task_id, &init.agent_name, &*agent_runner, &session_output);
799
800        // Write structured result.json for deterministic testing and debugging
801        response_handler::write_result_json(
802            &init.attempt_dir,
803            task_id,
804            &init.agent_name,
805            init.model_name.as_deref(),
806            session_output.exit_code,
807            init.new_attempts,
808            &parse_result,
809            &session_output.raw_stdout,
810            &session_output.raw_stderr,
811        )
812        .await;
813
814        // Handle outcome: success or error recovery
815        // `fallback_error` carries the attributed error message from `handle_error()` when
816        // the Continue path is taken.  It is passed as `error_override` to `build_run_audit`
817        // so that stale `last_error` values from a previous agent's run are never written
818        // into the current run's audit record (fixes misattribution bug: task_run shows
819        // agent=kimi but error="minimax timed out...").
820        let (final_status, push_failed, fallback_error) = match parse_result {
821            Ok(ref parsed) => {
822                let (status, push_failed) = response_handler::handle_success(
823                    task_id,
824                    parsed.clone(),
825                    &init.wt,
826                    &init.task_title,
827                    &init.agent_name,
828                    init.model_name.as_deref(),
829                    init.new_attempts,
830                    &self.repo,
831                    &self.store,
832                    &session_output.raw_stdout,
833                )
834                .await?;
835                (status, push_failed, None)
836            }
837            Err(ref agent_err) => {
838                match fallback::handle_error(
839                    task_id,
840                    agent_err,
841                    &init.agent_name,
842                    &*agent_runner,
843                    init.model_name.as_deref(),
844                    init.complexity.as_deref(),
845                    init.new_attempts,
846                    &self.store,
847                    &self.repo,
848                )
849                .await?
850                {
851                    fallback::ErrorHandleResult::EarlyReturn { status } => {
852                        // Task rerouted — clean up tmux session and env secrets
853                        session::cleanup_session(task_id, &tmux, &tmux_session).await;
854                        // The fallback handler already writes a sanitized `last_error`
855                        // into the store for EarlyReturn paths (reroutes / needs_review).
856                        // Passing the raw agent_err here would record noisy provider
857                        // payloads (e.g. full api_retry NDJSON). Leave error_override
858                        // as None so build_run_audit will prefer the store's last_error
859                        // (see src/engine/runner/fallback.rs which persists a compact
860                        // summary for rate-limit and reroute cases).
861                        let audit = self
862                            .build_run_audit(RunAuditInput {
863                                task_id,
864                                status: &status,
865                                parse_result: &parse_result,
866                                raw_stdout: &session_output.raw_stdout,
867                                raw_stderr: &session_output.raw_stderr,
868                                started_at,
869                                error_override: None,
870                                elapsed_secs: session_output.elapsed_secs,
871                                push_failed: false,
872                            })
873                            .await;
874                        return Ok(Some(RunExecution {
875                            status,
876                            exit_code: Some(session_output.exit_code),
877                            audit,
878                        }));
879                    }
880                    fallback::ErrorHandleResult::Continue { status, error } => {
881                        (status, false, Some(error))
882                    }
883                }
884            }
885        };
886
887        // Kill tmux session if still alive
888        session::cleanup_session(task_id, &tmux, &tmux_session).await;
889
890        // For no-op done tasks (agent finished with no code changes and no PR),
891        // clean up the worktree and branch immediately — tmux session is already
892        // dead so the tmux guard won't block removal.
893        if final_status.as_str() == "done" {
894            if let Some(ref st) = self.store {
895                if let Err(e) =
896                    crate::engine::cleanup::cleanup_task_worktree(task_id, &self.repo, st).await
897                {
898                    tracing::warn!(task_id, error = ?e, "worktree cleanup failed after no-op done");
899                }
900            }
901        }
902
903        // Return final status and the session exit code so callers can record it.
904        // Use `fallback_error` as error_override when available: it carries the error message
905        // attributed to the current agent (set by handle_error), preventing stale last_error
906        // values from a prior run from contaminating this run's audit record.
907        let audit = self
908            .build_run_audit(RunAuditInput {
909                task_id,
910                status: &final_status,
911                parse_result: &parse_result,
912                raw_stdout: &session_output.raw_stdout,
913                raw_stderr: &session_output.raw_stderr,
914                started_at,
915                error_override: fallback_error,
916                elapsed_secs: session_output.elapsed_secs,
917                push_failed,
918            })
919            .await;
920
921        Ok(Some(RunExecution {
922            status: final_status.to_string(),
923            exit_code: Some(session_output.exit_code),
924            audit,
925        }))
926    }
927
928    /// Record task execution metrics to the database.
929    #[allow(clippy::too_many_arguments)]
930    async fn record_metrics(
931        &self,
932        task_id: &str,
933        agent_name: &str,
934        model_name: &Option<String>,
935        route_result: &Option<RouteResult>,
936        started_at: &chrono::DateTime<Utc>,
937        attempts: u32,
938        final_status: &str,
939        error_type: Option<&str>,
940    ) {
941        let completed_at = Utc::now();
942        let duration_seconds = (completed_at - *started_at).num_milliseconds() as f64 / 1000.0;
943
944        // Classify outcome based on the final status and optional error_type/last_error
945        let outcome = match final_status {
946            "done" | "in_progress" | "in_review" => "success",
947            "new" => "rerouted",
948            "needs_review" => classify_run_error_type(error_type.unwrap_or("")),
949            _ => "unknown",
950        };
951
952        let complexity = route_result.as_ref().map(|r| r.complexity.clone());
953        let files_changed = git_ops::count_changed_files(&PathBuf::from(
954            self.get_field(task_id, "worktree")
955                .await
956                .unwrap_or_default(),
957        ))
958        .await
959        .unwrap_or(0);
960
961        if let Some(ref store) = self.store {
962            // Only set error_type for non-success outcomes
963            let db_error_type: Option<String> = if outcome == "success" {
964                None
965            } else {
966                Some(outcome.to_string())
967            };
968
969            // Read cost data from store
970            let usage = store::get_token_usage(&self.store, &self.repo, task_id).await;
971            let cost = store::get_cost_estimate(&self.store, &self.repo, task_id).await;
972            let input_tokens = if usage.input_tokens > 0 {
973                Some(usage.input_tokens as i64)
974            } else {
975                None
976            };
977            let output_tokens = if usage.output_tokens > 0 {
978                Some(usage.output_tokens as i64)
979            } else {
980                None
981            };
982            let input_cost = if cost.input_cost_usd > 0.0 {
983                Some(cost.input_cost_usd)
984            } else {
985                None
986            };
987            let output_cost = if cost.output_cost_usd > 0.0 {
988                Some(cost.output_cost_usd)
989            } else {
990                None
991            };
992            let total_cost = if cost.total_cost_usd > 0.0 {
993                Some(cost.total_cost_usd)
994            } else {
995                None
996            };
997
998            let metric = InsertTaskMetric {
999                repo: &self.repo,
1000                task_id,
1001                agent: agent_name,
1002                model: model_name.as_deref(),
1003                complexity: complexity.as_deref(),
1004                outcome,
1005                duration_seconds,
1006                started_at,
1007                completed_at: &completed_at,
1008                attempts: attempts as i32 + 1,
1009                files_changed: files_changed as i32,
1010                error_type: db_error_type.as_deref(),
1011                input_tokens,
1012                output_tokens,
1013                input_cost_usd: input_cost,
1014                output_cost_usd: output_cost,
1015                total_cost_usd: total_cost,
1016            };
1017
1018            if let Err(e) = store.insert_task_metric(&metric).await {
1019                tracing::error!(task_id, ?e, "failed to record task metrics");
1020            }
1021        }
1022    }
1023
1024    /// Run a task with full engine context (backend, tmux, capture).
1025    ///
1026    /// Called by the engine dispatch loop with richer context.
1027    /// Returns a `WeightSignal` for the engine to feed back to the router.
1028    pub async fn run_with_context(
1029        &self,
1030        task: &ExternalTask,
1031        backend: &Arc<dyn ExternalBackend>,
1032        _tmux: &Arc<TmuxManager>,
1033        route_result: Option<&RouteResult>,
1034    ) -> anyhow::Result<WeightSignal> {
1035        let task_id = &task.id.0;
1036        let agent = route_result.map(|r| r.agent.as_str());
1037        let agent_name = agent.unwrap_or("claude").to_string();
1038        let model = route_result.and_then(|r| r.model.as_deref());
1039
1040        // Record start time for metrics (before any work begins)
1041        let started_at = Utc::now();
1042
1043        // Store task info for prompt building (title/body already in store tasks table)
1044
1045        // Record run start in task_runs audit trail
1046        let run_audit_id = if let Some(ref store) = self.store {
1047            let attempt: i32 = self
1048                .get_field(task_id, "attempts")
1049                .await
1050                .and_then(|s| s.parse().ok())
1051                .unwrap_or(0)
1052                + 1;
1053            if let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, task_id).await {
1054                match store
1055                    .start_run(&crate::store::StartRun {
1056                        task_id: store_id,
1057                        attempt,
1058                        run_type: "agent",
1059                        agent: &agent_name,
1060                        model: model.unwrap_or(""),
1061                        command: &format!("{} --model {}", agent_name, model.unwrap_or("default")),
1062                        prompt: &task.body,
1063                    })
1064                    .await
1065                {
1066                    Ok(run_id) => Some(run_id),
1067                    Err(e) => {
1068                        tracing::warn!(
1069                            task_id,
1070                            error = %e,
1071                            "failed to record run start in audit trail"
1072                        );
1073                        None
1074                    }
1075                }
1076            } else {
1077                None
1078            }
1079        } else {
1080            None
1081        };
1082
1083        // Run the task.
1084        // Always finalize the run audit row, even on early-return/error paths.
1085        let run_result = match self
1086            .run(task_id, agent, model, Some(&**backend), &started_at)
1087            .await
1088        {
1089            Ok(run_result) => run_result,
1090            Err(e) => {
1091                if let Some(run_id) = run_audit_id {
1092                    if let Some(ref store) = self.store {
1093                        if let Err(complete_err) = store
1094                            .complete_run(&crate::store::CompleteRun {
1095                                run_id,
1096                                exit_code: Some(-1),
1097                                stdout: "",
1098                                stderr: "",
1099                                parsed: "",
1100                                outcome: "failed",
1101                                error: &e.to_string(),
1102                                tokens: crate::store::RunTokenUsage::default(),
1103                            })
1104                            .await
1105                        {
1106                            tracing::warn!(
1107                                task_id,
1108                                run_id,
1109                                error = %complete_err,
1110                                "failed to record run completion in audit trail after runner error"
1111                            );
1112                        }
1113                    }
1114                }
1115                return Err(e);
1116            }
1117        };
1118
1119        // If the runner guard skipped the task, do not re-post stale data as a new comment.
1120        let (status, exit_code_opt, run_audit) = match run_result {
1121            Some(result) => (result.status, result.exit_code, result.audit),
1122            None => {
1123                if let Some(run_id) = run_audit_id {
1124                    if let Some(ref store) = self.store {
1125                        if let Err(e) = store
1126                            .complete_run(&crate::store::CompleteRun {
1127                                run_id,
1128                                exit_code: Some(-1),
1129                                stdout: "",
1130                                stderr: "",
1131                                parsed: "",
1132                                outcome: "aborted",
1133                                error: "run skipped by guard",
1134                                tokens: crate::store::RunTokenUsage::default(),
1135                            })
1136                            .await
1137                        {
1138                            tracing::warn!(
1139                                task_id,
1140                                run_id,
1141                                error = %e,
1142                                "failed to record skipped run completion in audit trail"
1143                            );
1144                        }
1145                    }
1146                }
1147                tracing::info!(task_id, "guard skipped task — not posting stale result");
1148                return Ok(WeightSignal::None);
1149            }
1150        };
1151
1152        // Process delegations if the agent requested subtasks
1153        let delegations_raw = self
1154            .get_field(task_id, "delegations")
1155            .await
1156            .unwrap_or_default();
1157        if !delegations_raw.is_empty() {
1158            match serde_json::from_str::<Vec<crate::parser::Delegation>>(&delegations_raw) {
1159                Ok(delegations) if !delegations.is_empty() => {
1160                    self.process_delegations(task, &delegations, backend)
1161                        .await?;
1162                    if let Err(e) = store::store_set_result(
1163                        &self.store,
1164                        &self.repo,
1165                        task_id,
1166                        &[("delegations", serde_json::json!([]))],
1167                    )
1168                    .await
1169                    {
1170                        tracing::warn!(task_id, err = %e, "failed to clear delegations in store");
1171                    }
1172                }
1173                Ok(_) => {} // empty list, nothing to do
1174                Err(e) => {
1175                    tracing::error!(task_id, error = %e, "corrupt delegations JSON — clearing");
1176                    if let Err(e2) = store::store_set_result(
1177                        &self.store,
1178                        &self.repo,
1179                        task_id,
1180                        &[("delegations", serde_json::json!([]))],
1181                    )
1182                    .await
1183                    {
1184                        tracing::warn!(task_id, err = %e2, "failed to clear corrupt delegations in store");
1185                    }
1186                    if let Err(e2) = store::store_set_result(
1187                        &self.store,
1188                        &self.repo,
1189                        task_id,
1190                        &[(
1191                            "last_error",
1192                            serde_json::json!(format!("delegation parse failed: {e}")),
1193                        )],
1194                    )
1195                    .await
1196                    {
1197                        tracing::warn!(task_id, err = %e2, "failed to write delegation_parse_failed last_error to store");
1198                    }
1199                }
1200            }
1201        }
1202
1203        // Post result to GitHub
1204        let summary = self.get_field(task_id, "summary").await.unwrap_or_default();
1205        let last_error = self
1206            .get_field(task_id, "last_error")
1207            .await
1208            .unwrap_or_default();
1209
1210        // Determine weight signal based on outcome.
1211        // Any reroute ("new"/"routed") should avoid review-gate and go back through
1212        // normal dispatch, regardless of why it was rerouted.
1213        let is_rerouted = status == "new" || status == "routed";
1214        if is_rerouted {
1215            // Check if silence detection (tick phase1b) already re-routed this task while
1216            // the session was running. Silence detection sets status to Routed/New and
1217            // stores the fallback agent — we must NOT overwrite those fields here.
1218            // If the stored status is still InProgress, silence detection did not fire
1219            // and we can safely clear agent/model for normal reroute.
1220            let silence_already_reset = store::opt_store_get_task(&self.store, &self.repo, task_id)
1221                .await
1222                .map(|t| !matches!(t.status, crate::store::TaskStatus::InProgress))
1223                .unwrap_or(false);
1224
1225            if !silence_already_reset {
1226                if let Err(e) = store::store_set_result(
1227                    &self.store,
1228                    &self.repo,
1229                    task_id,
1230                    &[
1231                        ("agent", serde_json::json!("")),
1232                        ("model", serde_json::json!("")),
1233                    ],
1234                )
1235                .await
1236                {
1237                    tracing::warn!(task_id, err = %e, "failed to clear agent/model for reroute in store");
1238                }
1239            } else {
1240                tracing::debug!(
1241                    task_id,
1242                    "silence detection already set fallback agent — skipping agent/model clear"
1243                );
1244            }
1245        }
1246        // Determine weight signal based on outcome.
1247        // Reroutes should only emit RateLimited when the underlying error is genuinely
1248        // a rate limit (not silence detection, timeouts, parse errors, etc.). This prevents
1249        // false cooldown cascades where non-rate-limit failures are treated as rate limits.
1250        let is_rate_limit_error = classify_run_error_type(&last_error) == "rate_limit";
1251        let weight_signal = if is_rerouted {
1252            if is_rate_limit_error {
1253                WeightSignal::RateLimited {
1254                    agent: agent_name.clone(),
1255                }
1256            } else {
1257                // Non-rate-limit reroutes (silence detection, timeout, parse error): signal
1258                // the engine to keep the task in "new" for re-routing. Using Rerouted (not
1259                // None) prevents tick from converting the silence-reset back to needs_review,
1260                // which would trigger the no-code review shortcut and falsely mark it done.
1261                WeightSignal::Rerouted
1262            }
1263        } else if status == "needs_review" {
1264            // Rate limit errors that escalate to needs_review must NOT reset backoff
1265            // counters (issue #2153).  These errors indicate the agent is genuinely
1266            // unavailable — resetting would wipe the cooldown and cause the next dispatch
1267            // to retry the same rate-limited agent immediately.
1268            //
1269            // Detection: classify_run_error_type() confirms "rate limit" in last_error.
1270            // Reuse the same classifier as `is_rate_limit_error` above rather than a raw
1271            // substring check — billing-cycle exhaustion messages (e.g. Kimi's "usage
1272            // limit for this billing cycle") also contain "usage limit"/"rate limit", so
1273            // a bare substring check would record a rate-limit event into the router's
1274            // health signal for a condition that isn't a genuine rate limit (issue #3529).
1275            let has_rate_limit_error = classify_run_error_type(&last_error) == "rate_limit";
1276            if has_rate_limit_error {
1277                // Record in the metrics store so the router's health check detects it.
1278                if let Some(ref store) = self.store {
1279                    if let Err(e) = store
1280                        .record_rate_limit(&agent_name, "rate_limit", Some(task_id))
1281                        .await
1282                    {
1283                        tracing::warn!(task_id, agent = %agent_name, err = %e, "failed to persist rate_limit event — router health check may miss this signal");
1284                    }
1285                }
1286                // Also record via the cooldown system so model_for_complexity() skips
1287                // this model.  Use the stored last_error text as the message since
1288                // classify_run_error_type() already confirmed it contains "rate limit".
1289                crate::engine::cooldown::record_rate_limit(&agent_name, model, &last_error).await;
1290                WeightSignal::RateLimited {
1291                    agent: agent_name.clone(),
1292                }
1293            } else {
1294                // Non-rate-limit "needs_review" (e.g. agent successfully completed but
1295                // review agent is needed) — reset backoff so the next failure starts fresh.
1296                crate::engine::cooldown::record_agent_success(&agent_name, model.unwrap_or(""))
1297                    .await;
1298                WeightSignal::Success {
1299                    agent: agent_name.clone(),
1300                }
1301            }
1302        } else if status == "done" || status == "in_progress" || status == "in_review" {
1303            // Reset exponential backoff counters so the next failure starts fresh.
1304            crate::engine::cooldown::record_agent_success(&agent_name, model.unwrap_or("")).await;
1305            WeightSignal::Success {
1306                agent: agent_name.clone(),
1307            }
1308        } else if status == "blocked" {
1309            WeightSignal::Blocked
1310        } else {
1311            WeightSignal::None
1312        };
1313
1314        // Record metrics
1315        {
1316            let attempts: u32 = self
1317                .get_field(task_id, "attempts")
1318                .await
1319                .and_then(|s| s.parse().ok())
1320                .unwrap_or(0);
1321            self.record_metrics(
1322                task_id,
1323                &agent_name,
1324                &model.map(|s| s.to_string()),
1325                &route_result.cloned(),
1326                &started_at,
1327                attempts,
1328                &status,
1329                if last_error.is_empty() {
1330                    None
1331                } else {
1332                    Some(&last_error)
1333                },
1334            )
1335            .await;
1336        }
1337
1338        // Complete run in task_runs audit trail (include exit code from runner when available)
1339        if let Some(run_id) = run_audit_id {
1340            if let Some(ref store) = self.store {
1341                if let Err(e) = store
1342                    .complete_run(&crate::store::CompleteRun {
1343                        run_id,
1344                        exit_code: exit_code_opt,
1345                        stdout: &run_audit.stdout,
1346                        stderr: &run_audit.stderr,
1347                        parsed: &run_audit.parsed_response,
1348                        outcome: &run_audit.outcome,
1349                        error: &run_audit.error,
1350                        tokens: crate::store::RunTokenUsage {
1351                            input_tokens: run_audit.input_tokens,
1352                            output_tokens: run_audit.output_tokens,
1353                            total_cost_usd: run_audit.total_cost_usd,
1354                            duration_secs: run_audit.duration_secs,
1355                        },
1356                    })
1357                    .await
1358                {
1359                    tracing::warn!(
1360                        task_id,
1361                        run_id,
1362                        error = %e,
1363                        "failed to record run completion in audit trail"
1364                    );
1365                }
1366            }
1367        }
1368
1369        // If task was rerouted (status=routed or new after run), update GitHub agent label
1370        // so the router doesn't re-route back to the same failed agent.
1371        if is_rerouted && !is_internal_id(task_id) {
1372            let new_agent = self.get_field(task_id, "agent").await.unwrap_or_default();
1373            if !new_agent.is_empty() && new_agent != agent_name {
1374                update_agent_label_after_reroute(
1375                    &self.repo,
1376                    task,
1377                    backend,
1378                    &agent_name,
1379                    &new_agent,
1380                )
1381                .await;
1382            }
1383        }
1384
1385        // Internal tasks: skip GitHub status/comment updates — dispatch phase handles status.
1386        if is_internal_id(task_id) {
1387            return Ok(weight_signal);
1388        }
1389
1390        // Update GitHub status
1391        let new_status = match status.as_str() {
1392            "done" => Status::Done,
1393            "in_progress" => Status::InProgress,
1394            "in_review" => Status::InReview,
1395            "blocked" => Status::Blocked,
1396            "needs_review" => Status::NeedsReview,
1397            "routed" => Status::Routed, // Rerouted (agent/model already set)
1398            "new" => Status::New,       // Legacy reroute path
1399            _ => Status::NeedsReview,
1400        };
1401        // Store-first: update SQLite (must succeed), then mirror to backend.
1402        if let Some(ref store) = self.store {
1403            if let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &task.id.0).await {
1404                store
1405                    .update_status(
1406                        store_id,
1407                        crate::engine::tasks::status_to_task_status(new_status),
1408                    )
1409                    .await
1410                    .context("store-first status update failed in runner")?;
1411            }
1412        }
1413        if let Err(e) = backend.update_status(&task.id, new_status).await {
1414            tracing::warn!(
1415                task_id,
1416                ?new_status,
1417                err = %e,
1418                "failed to mirror runner status to backend — store is authoritative"
1419            );
1420        }
1421
1422        // Post comment (scan for secrets before posting to GitHub)
1423        let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ");
1424        let mut raw_comment = if !summary.is_empty() {
1425            format!("[{now}] {status}: {summary}")
1426        } else if !last_error.is_empty() {
1427            format!("[{now}] {status}: {last_error}")
1428        } else {
1429            format!("[{now}] {status}")
1430        };
1431
1432        // Append visible attribution footer (matches PR body style)
1433        let footer = crate::engine::attribution_footer("Created", &agent_name, model);
1434        raw_comment.push_str(&footer);
1435
1436        // Scan for leaked secrets and redact if needed
1437        let comment = if security::has_leaks(&raw_comment) {
1438            let leaks = security::scan(&raw_comment);
1439            let rules: Vec<&str> = leaks.iter().map(|l| l.rule).collect();
1440            let warning = format!(
1441                "\n\n> ⚠️ **Security Notice**: {} potential secret(s) detected and redacted: {}",
1442                leaks.len(),
1443                rules.join(", ")
1444            );
1445            let redacted = security::redact(&raw_comment);
1446            format!("{redacted}{warning}")
1447        } else {
1448            raw_comment
1449        };
1450        backend.post_comment(&task.id, &comment).await?;
1451
1452        Ok(weight_signal)
1453    }
1454
1455    /// Process delegations from an agent response.
1456    ///
1457    /// Creates child GitHub issues for each delegation and marks the parent
1458    /// as blocked. The engine's Phase 4 unblock mechanism will re-activate
1459    /// the parent when all children are done.
1460    async fn process_delegations(
1461        &self,
1462        parent_task: &ExternalTask,
1463        delegations: &[crate::parser::Delegation],
1464        backend: &Arc<dyn ExternalBackend>,
1465    ) -> anyhow::Result<()> {
1466        let parent_id = &parent_task.id;
1467
1468        // Fetch existing open tasks once for dedup — prevents creating duplicate
1469        // GitHub issues when the same delegations are produced on repeated runs.
1470        let existing_titles: std::collections::HashSet<String> =
1471            match backend.list_all_tasks().await {
1472                Ok(tasks) => tasks.into_iter().map(|t| t.title).collect(),
1473                Err(e) => {
1474                    tracing::warn!(err = %e, "failed to fetch tasks for dedup — will create all");
1475                    std::collections::HashSet::new()
1476                }
1477            };
1478
1479        for delegation in delegations {
1480            // Skip if an issue with the same title already exists
1481            if existing_titles.contains(&delegation.title) {
1482                tracing::info!(
1483                    parent = parent_id.0,
1484                    title = delegation.title,
1485                    "skipping delegation — issue with same title already exists"
1486                );
1487                continue;
1488            }
1489
1490            // Build labels: status:new + any labels from the delegation
1491            let mut labels = delegation.labels.clone();
1492            labels.push("status:new".to_string());
1493
1494            // Build child body with delegation reference
1495            let child_body = format!(
1496                "{}\n\n---\n_Delegated from #{}_{}",
1497                delegation.body,
1498                parent_id.0,
1499                crate::engine::orch_footer()
1500            );
1501
1502            match backend
1503                .create_sub_task(parent_id, &delegation.title, &child_body, &labels)
1504                .await
1505            {
1506                Ok(child_id) => {
1507                    tracing::info!(
1508                        parent = parent_id.0,
1509                        child = child_id.0,
1510                        title = delegation.title,
1511                        "created delegated subtask"
1512                    );
1513                }
1514                Err(e) => {
1515                    tracing::error!(
1516                        parent = parent_id.0,
1517                        title = delegation.title,
1518                        err = %e,
1519                        "failed to create delegated subtask"
1520                    );
1521                }
1522            }
1523        }
1524
1525        // Mark parent as blocked — store-first, then mirror to backend.
1526        if let Some(ref store) = self.store {
1527            if let Ok(Some(store_id)) = store.resolve_task_id(&self.repo, &parent_id.0).await {
1528                store
1529                    .update_status(store_id, crate::store::TaskStatus::Blocked)
1530                    .await
1531                    .context("store-first: failed to block parent in store")?;
1532            }
1533        }
1534        if let Err(e) = backend.update_status(parent_id, Status::Blocked).await {
1535            tracing::warn!(
1536                parent = parent_id.0,
1537                err = %e,
1538                "failed to mirror blocked status to backend — store is authoritative"
1539            );
1540        }
1541
1542        // Post summary comment on parent
1543        let summary = delegations
1544            .iter()
1545            .enumerate()
1546            .map(|(i, d)| format!("{}. {}", i + 1, d.title))
1547            .collect::<Vec<_>>()
1548            .join("\n");
1549
1550        backend
1551            .post_comment(
1552                parent_id,
1553                &format!(
1554                    "Delegated {} subtask(s):\n\n{}\n\nParent task is blocked until all subtasks complete.{}",
1555                    delegations.len(),
1556                    summary,
1557                    crate::engine::orch_footer()
1558                ),
1559            )
1560            .await?;
1561
1562        Ok(())
1563    }
1564
1565    /// Resolve the project directory for this repo.
1566    async fn resolve_project_dir(&self) -> anyhow::Result<PathBuf> {
1567        // Explicit env var always wins
1568        if let Ok(dir) = std::env::var("PROJECT_DIR") {
1569            if !dir.is_empty() {
1570                return Ok(PathBuf::from(dir));
1571            }
1572        }
1573
1574        // Check projects list in global config — find path for this repo
1575        if let Ok(paths) = config::get_project_paths() {
1576            for path_str in &paths {
1577                let path = PathBuf::from(path_str);
1578                // Check if this project's .orch.yml has matching repo
1579                if let Ok(repo) = config::get_repo_for_project(&path) {
1580                    // Use tokio::fs::metadata for async-safe existence check
1581                    let path_for_check = path.clone();
1582                    if repo == self.repo && tokio::fs::metadata(&path_for_check).await.is_ok() {
1583                        return Ok(path);
1584                    }
1585                }
1586            }
1587        }
1588
1589        // Legacy: check config project_dir
1590        if let Ok(dir) = config::get("project_dir") {
1591            if !dir.is_empty() {
1592                let path = PathBuf::from(&dir);
1593                let path_for_check = path.clone();
1594                if tokio::fs::metadata(&path_for_check).await.is_ok() {
1595                    return Ok(path);
1596                }
1597            }
1598        }
1599
1600        // Check for bare clone
1601        let parts: Vec<&str> = self.repo.split('/').collect();
1602        if parts.len() == 2 {
1603            let bare = self
1604                .orch_home
1605                .join("projects")
1606                .join(parts[0])
1607                .join(format!("{}.git", parts[1]));
1608            let bare_for_check = bare.clone();
1609            if tokio::task::spawn_blocking(move || bare_for_check.exists())
1610                .await
1611                .unwrap_or(false)
1612            {
1613                return Ok(bare);
1614            }
1615        }
1616
1617        // No resolvable project directory: not in active config, no legacy
1618        // project_dir, no bare clone. Falling back to the service's own CWD
1619        // here would silently point worktree creation at an unrelated
1620        // directory (see issue #3416) — surface a clear error instead.
1621        anyhow::bail!(
1622            "no resolvable project directory for repo {} — not in active config, no bare clone at {}, no legacy project_dir set",
1623            self.repo,
1624            self.orch_home
1625                .join("projects")
1626                .join(self.repo.split('/').next().unwrap_or_default())
1627                .join(format!(
1628                    "{}.git",
1629                    self.repo.split('/').nth(1).unwrap_or_default()
1630                ))
1631                .display()
1632        )
1633    }
1634}
1635
1636/// Update the GitHub agent label on an external task when it is rerouted to a new agent.
1637///
1638/// Removes the old `agent:<old_agent>` label, ensures the new `agent:<new_agent>` label exists
1639/// in the repo, then applies it. Returns `true` only when both `ensure_label` (repo label creation)
1640/// and `set_labels` (issue label assignment) succeed. `remove_label` failures are non-fatal.
1641async fn update_agent_label_after_reroute(
1642    #[cfg_attr(test, allow(unused_variables))] repo: &str,
1643    task: &ExternalTask,
1644    backend: &Arc<dyn ExternalBackend>,
1645    old_agent: &str,
1646    new_agent: &str,
1647) -> bool {
1648    let old_label = format!("agent:{old_agent}");
1649    if let Err(e) = backend.remove_label(&task.id, &old_label).await {
1650        tracing::warn!(
1651            task_id = ?task.id,
1652            label = old_label,
1653            error = %e,
1654            "failed to remove old agent label during re-route"
1655        );
1656    }
1657    let new_label = format!("agent:{new_agent}");
1658
1659    // ensure_label creates the repo-level label if missing (best-effort).
1660    // Skipped under #[cfg(test)] to avoid real GitHub API calls.
1661    #[cfg(not(test))]
1662    {
1663        match crate::github::http::GhHttp::new() {
1664            Ok(gh) => {
1665                if gh
1666                    .ensure_label(
1667                        repo,
1668                        &new_label,
1669                        crate::github::http::status_label_color(&new_label),
1670                        &format!("Agent: {new_agent}"),
1671                    )
1672                    .await
1673                    .is_err()
1674                {
1675                    tracing::warn!(
1676                        task_id = ?task.id,
1677                        label = %new_label,
1678                        "ensure_label failed during agent label update after failover"
1679                    );
1680                }
1681            }
1682            Err(e) => {
1683                tracing::warn!(
1684                    task_id = ?task.id,
1685                    err = %e,
1686                    "GhHttp::new() failed — skipping ensure_label"
1687                );
1688            }
1689        }
1690    }
1691
1692    if let Err(e) = backend
1693        .set_labels(&task.id, std::slice::from_ref(&new_label))
1694        .await
1695    {
1696        tracing::warn!(
1697            task_id = ?task.id,
1698            label = %new_label,
1699            error = %e,
1700            "set_labels failed during agent label update after failover"
1701        );
1702        return false;
1703    }
1704    true
1705}
1706
1707#[cfg(test)]
1708mod tests {
1709    use super::*;
1710    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
1711    use crate::engine::runner::agents::patterns::safe_tail as safe_utf8_tail;
1712    use crate::parser::AgentResponse;
1713    use crate::store::{NewTask, TaskStore};
1714    use async_trait::async_trait;
1715    use chrono::Utc;
1716    use once_cell::sync::Lazy;
1717    use std::sync::{Arc, Mutex};
1718    use tempfile::TempDir;
1719
1720    static ENV_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
1721
1722    fn ok_parse_result(status: &str) -> Result<agents::ParsedResponse, agents::AgentError> {
1723        Ok(agents::ParsedResponse {
1724            response: AgentResponse {
1725                status: status.to_string(),
1726                summary: String::new(),
1727                accomplished: vec![],
1728                remaining: vec![],
1729                files: vec![],
1730                error: None,
1731                input_tokens: None,
1732                output_tokens: None,
1733                learnings: vec![],
1734                delegations: vec![],
1735            },
1736            input_tokens: None,
1737            output_tokens: None,
1738            duration_ms: None,
1739        })
1740    }
1741
1742    // ── safe_utf8_tail ───────────────────────────────────────────────────────
1743
1744    #[test]
1745    fn safe_utf8_tail_short_string_returned_as_is() {
1746        assert_eq!(safe_utf8_tail("hello", 100), "hello");
1747        assert_eq!(safe_utf8_tail("", 10), "");
1748    }
1749
1750    #[test]
1751    fn safe_utf8_tail_truncates_ascii() {
1752        let s = "abcdefghij"; // 10 bytes
1753        assert_eq!(safe_utf8_tail(s, 5), "fghij");
1754        assert_eq!(safe_utf8_tail(s, 10), "abcdefghij");
1755    }
1756
1757    #[test]
1758    fn safe_utf8_tail_handles_multibyte_boundary() {
1759        // "日本語" is 3 chars × 3 bytes each = 9 bytes total
1760        let s = "日本語";
1761        assert_eq!(s.len(), 9);
1762        // Cutting at 8 bytes lands in the middle of "語" (3 bytes at 6..9).
1763        // safe_utf8_tail should walk forward to the next boundary at 9 (end) rather
1764        // than panicking on an invalid slice.
1765        let tail = safe_utf8_tail(s, 8);
1766        assert!(
1767            s.ends_with(tail),
1768            "tail must be a valid suffix of the original"
1769        );
1770        assert!(
1771            std::str::from_utf8(tail.as_bytes()).is_ok(),
1772            "tail must be valid UTF-8"
1773        );
1774    }
1775
1776    #[test]
1777    fn safe_utf8_tail_handles_mixed_ascii_and_multibyte() {
1778        let s = "hello日本語world"; // ascii + CJK + ascii
1779        for max_bytes in [1, 3, 5, 7, 11, 13, 20, 100] {
1780            let tail = safe_utf8_tail(s, max_bytes);
1781            assert!(
1782                std::str::from_utf8(tail.as_bytes()).is_ok(),
1783                "max_bytes={max_bytes}: tail must be valid UTF-8"
1784            );
1785            assert!(
1786                s.ends_with(tail),
1787                "max_bytes={max_bytes}: tail must be a suffix of original"
1788            );
1789        }
1790    }
1791
1792    #[test]
1793    fn safe_utf8_tail_exact_boundary() {
1794        let s = "abcdef"; // 6 bytes
1795        assert_eq!(safe_utf8_tail(s, 6), "abcdef"); // exactly fits
1796        assert_eq!(safe_utf8_tail(s, 0), ""); // empty tail
1797    }
1798
1799    #[test]
1800    fn parse_success_output_synthesizes_done_from_plain_text() {
1801        let runner = agents::get_runner("claude");
1802        let parsed = parse_success_output(
1803            "123",
1804            "claude",
1805            &*runner,
1806            "No open positions, no trade executions, no conditions change.",
1807        )
1808        .unwrap();
1809
1810        assert_eq!(parsed.response.status, "done");
1811        assert_eq!(
1812            parsed.response.summary,
1813            "No open positions, no trade executions, no conditions change."
1814        );
1815    }
1816
1817    #[test]
1818    fn parse_success_output_synthesizes_needs_review_from_error_text() {
1819        let runner = agents::get_runner("claude");
1820        let parsed =
1821            parse_success_output("123", "claude", &*runner, "Failed to update branch").unwrap();
1822
1823        assert_eq!(parsed.response.status, "needs_review");
1824        assert_eq!(
1825            parsed.response.error.as_deref(),
1826            Some("agent returned plain text instead of JSON")
1827        );
1828    }
1829
1830    #[test]
1831    fn parse_success_output_keeps_empty_text_invalid() {
1832        let runner = agents::get_runner("claude");
1833        let err = parse_success_output("123", "claude", &*runner, "").unwrap_err();
1834
1835        assert!(matches!(err, agents::AgentError::InvalidResponse { .. }));
1836    }
1837
1838    #[test]
1839    fn parse_success_output_preserves_tokens_from_ndjson() {
1840        // Simulates NDJSON output from --output-format stream-json where
1841        // the result event contains token usage. parse_response fails on
1842        // NDJSON (expects single JSON blob), but find_agent_result extracts
1843        // tokens from the NDJSON envelope so cost tracking is preserved.
1844        let ndjson = r#"{"type":"system","subtype":"init","session_id":"abc"}
1845{"type":"assistant","message":{"content":[{"type":"text","text":"Done."}],"usage":{"input_tokens":100,"output_tokens":50}}}
1846{"type":"result","subtype":"success","is_error":false,"duration_ms":5000,"result":"All changes committed successfully","usage":{"input_tokens":1234,"output_tokens":567}}"#;
1847        let runner = agents::get_runner("claude");
1848        let parsed = parse_success_output("456", "claude", &*runner, ndjson).unwrap();
1849
1850        assert_eq!(parsed.response.status, "done");
1851        assert_eq!(parsed.input_tokens, Some(1234));
1852        assert_eq!(parsed.output_tokens, Some(567));
1853        assert_eq!(parsed.duration_ms, Some(5000));
1854    }
1855
1856    #[test]
1857    fn parse_success_output_preserves_tokens_from_minimax_ndjson() {
1858        // MiniMax/Kimi use assistant event fallback when there's no result event.
1859        let ndjson = r#"{"type":"system","subtype":"init"}
1860{"type":"assistant","message":{"content":[{"type":"text","text":"No open positions."}],"usage":{"input_tokens":2000,"output_tokens":300}}}"#;
1861        let runner = agents::get_runner("minimax");
1862        let parsed = parse_success_output("789", "minimax", &*runner, ndjson).unwrap();
1863
1864        assert_eq!(parsed.response.status, "done");
1865        assert_eq!(parsed.input_tokens, Some(2000));
1866        assert_eq!(parsed.output_tokens, Some(300));
1867    }
1868
1869    #[test]
1870    fn classify_run_error_type_recognizes_specific_patterns() {
1871        let cases = [
1872            ("push failed: remote rejected", "push_failed"),
1873            ("No commits between main and branch", "no_commits"),
1874            ("create PR failed: 422", "pr_failed"),
1875            ("invalid response from agent", "parse_error"),
1876            ("parse error: missing json", "parse_error"),
1877            ("exceeded max attempts while rerouting", "max_attempts"),
1878            ("unit test failed", "ci_failure"),
1879            ("build failed: type mismatch", "ci_failure"),
1880            ("lint failed on warning", "ci_failure"),
1881            ("3 tests failed", "ci_failure"),
1882            ("test suite failed: 3 tests failed", "ci_failure"),
1883        ];
1884
1885        for (input, expected) in cases {
1886            assert_eq!(classify_run_error_type(input), expected, "input={input}");
1887        }
1888    }
1889
1890    #[test]
1891    fn classify_run_error_type_falls_back_for_unknown_errors() {
1892        assert_eq!(classify_run_error_type("something unexpected"), "failed");
1893    }
1894
1895    #[test]
1896    fn classify_run_error_type_connection_error_mentioning_pull_request_is_not_pr_failed() {
1897        // A generic GitHub API or connection error whose message happens to contain the
1898        // substring "pull request" must NOT be classified as pr_failed.
1899        assert_eq!(
1900            classify_run_error_type("failed to fetch pull request details from GitHub"),
1901            "failed"
1902        );
1903        assert_eq!(
1904            classify_run_error_type("connection reset while reading pull request data"),
1905            "failed"
1906        );
1907    }
1908
1909    #[test]
1910    fn classify_run_error_type_url_with_test_substring_is_not_ci_failure() {
1911        // A connection error whose URL path contains "/test-org/" must NOT be ci_failure.
1912        assert_eq!(
1913            classify_run_error_type(
1914                "failed to connect to api.github.com/repos/test-org/repo/pulls/123"
1915            ),
1916            "failed"
1917        );
1918        // Bare "test" word in an unrelated error must not trigger ci_failure.
1919        assert_eq!(
1920            classify_run_error_type("context deadline exceeded testing connection"),
1921            "failed"
1922        );
1923    }
1924
1925    #[test]
1926    fn classify_run_error_type_usage_without_limit_is_not_rate_limit() {
1927        // "usage" alone (e.g. token usage reporting) must NOT be rate_limit.
1928        assert_eq!(
1929            classify_run_error_type("token usage: 1245 tokens"),
1930            "failed"
1931        );
1932        assert_eq!(
1933            classify_run_error_type("incorrect usage of git command"),
1934            "failed"
1935        );
1936        // Only "usage limit" should trigger rate_limit.
1937        assert_eq!(
1938            classify_run_error_type("exceeded usage limit for this billing period"),
1939            "rate_limit"
1940        );
1941    }
1942
1943    #[test]
1944    fn classify_run_error_type_billing_cycle_is_not_auth_error() {
1945        // "billing cycle" quota-exhaustion messages must be classified as billing_cycle_exhausted,
1946        // not auth_error — aligning with cooldown.rs CreditExhaustionReason::BillingCycleExhausted.
1947        assert_eq!(
1948            classify_run_error_type(
1949                "You've reached your usage limit for this billing cycle. \
1950                 Your quota will be refreshed in the next cycle."
1951            ),
1952            "billing_cycle_exhausted"
1953        );
1954        assert_eq!(
1955            classify_run_error_type("quota exhausted for billing cycle"),
1956            "billing_cycle_exhausted"
1957        );
1958        // Plain "billing" without "cycle" that used to map to auth_error now falls through to "failed".
1959        assert_eq!(
1960            classify_run_error_type("billing account suspended"),
1961            "failed"
1962        );
1963    }
1964
1965    #[test]
1966    fn classify_run_error_type_monthly_quota_is_billing_cycle() {
1967        // Regression test for issue #3302: GitHub Copilot "You have exceeded your
1968        // monthly quota" must map to billing_cycle_exhausted, not "failed".
1969        assert_eq!(
1970            classify_run_error_type(
1971                "opencode failed: agent failed: You have exceeded your monthly quota"
1972            ),
1973            "billing_cycle_exhausted"
1974        );
1975        assert_eq!(
1976            classify_run_error_type("You have exceeded your monthly quota"),
1977            "billing_cycle_exhausted"
1978        );
1979    }
1980
1981    #[test]
1982    fn classify_run_outcome_routed_is_success() {
1983        // "routed" is a canonical non-success status (task will be re-dispatched).
1984        // It should be classified as "success" so the audit trail distinguishes it
1985        // from genuine failures (#2801).
1986        let parse_result = ok_parse_result("routed");
1987        assert_eq!(
1988            classify_run_outcome("routed", &parse_result, false),
1989            "success"
1990        );
1991    }
1992
1993    #[test]
1994    fn classify_run_outcome_blocked_is_blocked() {
1995        let parse_result = ok_parse_result("blocked");
1996        assert_eq!(
1997            classify_run_outcome("blocked", &parse_result, false),
1998            "blocked"
1999        );
2000    }
2001
2002    #[test]
2003    fn classify_run_outcome_non_canonical_is_failed() {
2004        // Non-canonical statuses should be classified as "failed" in the audit.
2005        // The explicit error message is provided by build_run_audit.
2006        let parse_result = ok_parse_result("fix_deployed");
2007        assert_eq!(
2008            classify_run_outcome("fix_deployed", &parse_result, false),
2009            "failed"
2010        );
2011    }
2012
2013    #[test]
2014    fn classify_run_outcome_kimi_billing_cycle_is_not_rate_limit() {
2015        // Regression test for issue #3529: Kimi's exact billing-cycle exhaustion
2016        // message was stored as outcome="rate_limit" because the generic "usage
2017        // limit" pattern in detect_rate_limit() matched before credit-exhaustion
2018        // detection ever ran. The outcome must reflect the actual failure mode.
2019        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2020            Err(agents::AgentError::RateLimit {
2021                message: "Failed to authenticate. API Error: 403 You've reached your usage \
2022                          limit for this billing cycle. Your quota will be refreshed soon."
2023                    .to_string(),
2024            });
2025        assert_eq!(
2026            classify_run_outcome("needs_review", &parse_result, false),
2027            "billing_cycle_exhausted"
2028        );
2029    }
2030
2031    #[test]
2032    fn classify_run_outcome_generic_rate_limit_is_unaffected() {
2033        // A genuine rate limit (no billing-cycle/credit-exhaustion markers) must
2034        // still classify as "rate_limit".
2035        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2036            Err(agents::AgentError::RateLimit {
2037                message: "429 Too Many Requests".to_string(),
2038            });
2039        assert_eq!(
2040            classify_run_outcome("needs_review", &parse_result, false),
2041            "rate_limit"
2042        );
2043    }
2044
2045    #[test]
2046    fn classify_run_outcome_auth_billing_cycle_is_not_failed() {
2047        // Auth-classified billing-cycle exhaustion (e.g. "billing cycle exhausted"
2048        // matches detect_auth_error's pattern list) must also be distinguished from
2049        // a generic auth failure.
2050        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2051            Err(agents::AgentError::Auth {
2052                message: "billing cycle exhausted for this account".to_string(),
2053            });
2054        assert_eq!(
2055            classify_run_outcome("needs_review", &parse_result, false),
2056            "billing_cycle_exhausted"
2057        );
2058    }
2059
2060    #[test]
2061    fn classify_run_outcome_generic_auth_error_is_unaffected() {
2062        let parse_result: Result<agents::ParsedResponse, agents::AgentError> =
2063            Err(agents::AgentError::Auth {
2064                message: "401 Unauthorized".to_string(),
2065            });
2066        assert_eq!(
2067            classify_run_outcome("needs_review", &parse_result, false),
2068            "failed"
2069        );
2070    }
2071
2072    #[tokio::test]
2073    async fn build_run_audit_success_ignores_stale_last_error() {
2074        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2075        let task_id = store
2076            .create(&NewTask {
2077                repo: "owner/repo".to_string(),
2078                origin: "internal".to_string(),
2079                title: "Test".to_string(),
2080                ..Default::default()
2081            })
2082            .await
2083            .unwrap();
2084        store
2085            .set_fields(
2086                task_id,
2087                &[("last_error", serde_json::json!("no error info available"))],
2088            )
2089            .await
2090            .unwrap();
2091
2092        let runner = TaskRunner::new("owner/repo".to_string()).with_store(store);
2093        let parse_result = ok_parse_result("done");
2094        let started_at = Utc::now();
2095        let audit = runner
2096            .build_run_audit(RunAuditInput {
2097                task_id: &task_id.to_string(),
2098                status: "done",
2099                parse_result: &parse_result,
2100                raw_stdout: "",
2101                raw_stderr: "",
2102                started_at: &started_at,
2103                error_override: None,
2104                elapsed_secs: Some(1),
2105                push_failed: false,
2106            })
2107            .await;
2108
2109        assert_eq!(audit.outcome, "success");
2110        assert!(audit.error.is_empty());
2111    }
2112
2113    #[tokio::test]
2114    async fn build_run_audit_non_success_uses_fallback_error_when_empty() {
2115        let runner = TaskRunner::new("owner/repo".to_string());
2116        let parse_result = ok_parse_result("new");
2117        let started_at = Utc::now();
2118        let audit = runner
2119            .build_run_audit(RunAuditInput {
2120                task_id: "1",
2121                status: "new",
2122                parse_result: &parse_result,
2123                raw_stdout: "",
2124                raw_stderr: "",
2125                started_at: &started_at,
2126                error_override: None,
2127                elapsed_secs: Some(1),
2128                push_failed: false,
2129            })
2130            .await;
2131
2132        assert_eq!(audit.outcome, "failed");
2133        // "new" is a canonical non-success status — no error message needed.
2134        assert_eq!(audit.error, "");
2135    }
2136
2137    #[tokio::test]
2138    async fn build_run_audit_non_canonical_status_reports_explicit_error() {
2139        // Non-canonical statuses (not in the allowlist) with no parse error
2140        // should get an explicit "unrecognized status: X" message instead of
2141        // the opaque "no error info available" placeholder (#2801).
2142        let runner = TaskRunner::new("owner/repo".to_string());
2143        let parse_result = ok_parse_result("fix_deployed");
2144        let started_at = Utc::now();
2145        let audit = runner
2146            .build_run_audit(RunAuditInput {
2147                task_id: "1",
2148                status: "fix_deployed",
2149                parse_result: &parse_result,
2150                raw_stdout: "",
2151                raw_stderr: "",
2152                started_at: &started_at,
2153                error_override: None,
2154                elapsed_secs: Some(1),
2155                push_failed: false,
2156            })
2157            .await;
2158
2159        assert_eq!(audit.outcome, "failed");
2160        assert_eq!(audit.error, "unrecognized status: fix_deployed");
2161    }
2162
2163    #[tokio::test]
2164    async fn build_run_audit_canonical_non_success_status_no_error_message() {
2165        // Canonical non-success statuses (e.g. "routed") should not produce
2166        // an error message — they're expected outcomes, not failures.
2167        for status in &["routed"] {
2168            let runner = TaskRunner::new("owner/repo".to_string());
2169            let parse_result = ok_parse_result(status);
2170            let started_at = Utc::now();
2171            let audit = runner
2172                .build_run_audit(RunAuditInput {
2173                    task_id: "1",
2174                    status,
2175                    parse_result: &parse_result,
2176                    raw_stdout: "",
2177                    raw_stderr: "",
2178                    started_at: &started_at,
2179                    error_override: None,
2180                    elapsed_secs: Some(1),
2181                    push_failed: false,
2182                })
2183                .await;
2184
2185            assert_eq!(audit.outcome, "success", "status={status}");
2186            assert_eq!(
2187                audit.error, "",
2188                "status={status}: no error for canonical status"
2189            );
2190        }
2191    }
2192
2193    #[tokio::test]
2194    async fn build_run_audit_blocked_uses_summary_when_error_missing() {
2195        let runner = TaskRunner::new("owner/repo".to_string());
2196        let parse_result = Ok(agents::ParsedResponse {
2197            response: AgentResponse {
2198                status: "blocked".to_string(),
2199                summary: "Could not proceed due to missing credentials".to_string(),
2200                accomplished: vec![],
2201                remaining: vec![],
2202                files: vec![],
2203                error: None,
2204                input_tokens: None,
2205                output_tokens: None,
2206                learnings: vec![],
2207                delegations: vec![],
2208            },
2209            input_tokens: None,
2210            output_tokens: None,
2211            duration_ms: None,
2212        });
2213        let started_at = Utc::now();
2214        let audit = runner
2215            .build_run_audit(RunAuditInput {
2216                task_id: "1",
2217                status: "blocked",
2218                parse_result: &parse_result,
2219                raw_stdout: "",
2220                raw_stderr: "",
2221                started_at: &started_at,
2222                error_override: None,
2223                elapsed_secs: Some(1),
2224                push_failed: false,
2225            })
2226            .await;
2227
2228        assert_eq!(audit.outcome, "blocked");
2229        assert_eq!(audit.error, "Could not proceed due to missing credentials");
2230    }
2231
2232    // ── resolve_project_dir ──────────────────────────────────────────────────
2233
2234    #[tokio::test]
2235    async fn resolve_project_dir_uses_project_dir_env() {
2236        // When PROJECT_DIR env var is set, it should always win.
2237        let runner = TaskRunner::new("owner/repo".to_string());
2238
2239        // Use a temp dir so the path exists
2240        let dir = tempfile::tempdir().unwrap();
2241        let dir_str = dir.path().to_string_lossy().to_string();
2242
2243        // Temporarily set the env var — note: tests run in parallel so we use a
2244        // dedicated key that only this test touches. std::env::set_var is not
2245        // thread-safe in general, but the subsequent call is synchronous and
2246        // isolated to the temp dir created above.
2247        std::env::set_var("PROJECT_DIR", &dir_str);
2248        let result = runner.resolve_project_dir().await;
2249        std::env::remove_var("PROJECT_DIR");
2250
2251        assert!(
2252            result.is_ok(),
2253            "resolve_project_dir should succeed with PROJECT_DIR set"
2254        );
2255        assert_eq!(result.unwrap(), dir.path());
2256    }
2257
2258    #[tokio::test]
2259    async fn resolve_project_dir_empty_project_dir_env_falls_through() {
2260        // An empty PROJECT_DIR should be ignored and fall through to other logic.
2261        let runner = TaskRunner::new("owner/testrepo-nonexistent".to_string());
2262        std::env::set_var("PROJECT_DIR", "");
2263        let result = runner.resolve_project_dir().await;
2264        std::env::remove_var("PROJECT_DIR");
2265
2266        // No active config entry, no legacy project_dir, no bare clone for this
2267        // repo: should return an explicit error rather than falling back to the
2268        // service's own CWD (see issue #3416).
2269        assert!(
2270            result.is_err(),
2271            "resolve_project_dir should error when no project dir can be resolved"
2272        );
2273        let err = result.unwrap_err().to_string();
2274        assert!(
2275            err.contains("owner/testrepo-nonexistent"),
2276            "error should name the unresolvable repo: {err}"
2277        );
2278    }
2279
2280    // ── mock backend for process_delegations ─────────────────────────────────
2281
2282    struct TrackingBackend {
2283        sub_tasks_created: Arc<Mutex<Vec<(String, String)>>>, // (title, parent_id)
2284        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
2285        comments: Arc<Mutex<Vec<(String, String)>>>, // (id, body)
2286    }
2287
2288    impl TrackingBackend {
2289        fn new() -> Self {
2290            Self {
2291                sub_tasks_created: Arc::new(Mutex::new(vec![])),
2292                status_updates: Arc::new(Mutex::new(vec![])),
2293                comments: Arc::new(Mutex::new(vec![])),
2294            }
2295        }
2296    }
2297
2298    #[async_trait]
2299    impl crate::backends::ExternalBackend for TrackingBackend {
2300        fn name(&self) -> &str {
2301            "tracking"
2302        }
2303        async fn create_task(
2304            &self,
2305            _t: &str,
2306            _b: &str,
2307            _l: &[String],
2308        ) -> anyhow::Result<ExternalId> {
2309            Ok(ExternalId("new".to_string()))
2310        }
2311        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2312            Ok(ExternalTask {
2313                id: id.clone(),
2314                title: "".to_string(),
2315                body: "".to_string(),
2316                state: "open".to_string(),
2317                labels: vec![],
2318                author: "".to_string(),
2319                created_at: "2026-01-01T00:00:00Z".to_string(),
2320                updated_at: "2026-01-01T00:00:00Z".to_string(),
2321                url: "".to_string(),
2322            })
2323        }
2324        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
2325            Ok(vec![])
2326        }
2327        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2328            Ok(vec![])
2329        }
2330        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2331            self.comments
2332                .lock()
2333                .unwrap()
2334                .push((id.0.clone(), body.to_string()));
2335            Ok(())
2336        }
2337        async fn set_labels(&self, _id: &ExternalId, _l: &[String]) -> anyhow::Result<()> {
2338            Ok(())
2339        }
2340        async fn remove_label(&self, _id: &ExternalId, _l: &str) -> anyhow::Result<()> {
2341            Ok(())
2342        }
2343        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2344            Ok(vec![])
2345        }
2346        async fn create_sub_task(
2347            &self,
2348            parent: &ExternalId,
2349            title: &str,
2350            _body: &str,
2351            _l: &[String],
2352        ) -> anyhow::Result<ExternalId> {
2353            self.sub_tasks_created
2354                .lock()
2355                .unwrap()
2356                .push((title.to_string(), parent.0.clone()));
2357            Ok(ExternalId(format!("child-{}", title)))
2358        }
2359        async fn ensure_status_label(&self, _l: &str) -> anyhow::Result<()> {
2360            Ok(())
2361        }
2362        async fn has_open_issue_with_title(&self, _t: &str, _l: &str) -> anyhow::Result<bool> {
2363            Ok(false)
2364        }
2365        async fn health_check(&self) -> anyhow::Result<()> {
2366            Ok(())
2367        }
2368        async fn is_pr_merged(&self, _b: &str) -> anyhow::Result<bool> {
2369            Ok(false)
2370        }
2371        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2372            Ok(None)
2373        }
2374        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
2375            Ok(vec![])
2376        }
2377        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2378            self.status_updates
2379                .lock()
2380                .unwrap()
2381                .push((id.0.clone(), status));
2382            Ok(())
2383        }
2384    }
2385
2386    fn make_task(id: &str) -> ExternalTask {
2387        ExternalTask {
2388            id: ExternalId(id.to_string()),
2389            title: format!("Task {id}"),
2390            body: "".to_string(),
2391            state: "open".to_string(),
2392            labels: vec![],
2393            author: "bot".to_string(),
2394            created_at: "2026-01-01T00:00:00Z".to_string(),
2395            updated_at: "2026-01-01T00:00:00Z".to_string(),
2396            url: "".to_string(),
2397        }
2398    }
2399
2400    // ── process_delegations ───────────────────────────────────────────────────
2401
2402    #[allow(clippy::await_holding_lock)]
2403    #[tokio::test]
2404    async fn process_delegations_creates_subtasks_and_blocks_parent() {
2405        let _guard = ENV_LOCK.lock().unwrap();
2406        let temp_home = TempDir::new().unwrap();
2407        let orch_home = temp_home.path().join(".orch");
2408        tokio::fs::create_dir_all(&orch_home).await.unwrap();
2409        let old_orch_home = std::env::var("ORCH_HOME").ok();
2410        std::env::set_var("ORCH_HOME", &orch_home);
2411
2412        let runner = TaskRunner::new("owner/repo".to_string());
2413        let parent = make_task("99");
2414        let backend = Arc::new(TrackingBackend::new());
2415        let backend_dyn: Arc<dyn crate::backends::ExternalBackend> = backend.clone();
2416
2417        let delegations = vec![
2418            crate::parser::Delegation {
2419                title: "Subtask A".to_string(),
2420                body: "Do A".to_string(),
2421                labels: vec!["enhancement".to_string()],
2422            },
2423            crate::parser::Delegation {
2424                title: "Subtask B".to_string(),
2425                body: "Do B".to_string(),
2426                labels: vec![],
2427            },
2428        ];
2429
2430        runner
2431            .process_delegations(&parent, &delegations, &backend_dyn)
2432            .await
2433            .unwrap();
2434
2435        // Two sub-tasks should have been created
2436        let created = backend.sub_tasks_created.lock().unwrap();
2437        assert_eq!(created.len(), 2);
2438        assert_eq!(created[0].1, "99", "parent id should be 99");
2439        assert!(
2440            created.iter().any(|(t, _)| t == "Subtask A"),
2441            "Subtask A should be created"
2442        );
2443        assert!(
2444            created.iter().any(|(t, _)| t == "Subtask B"),
2445            "Subtask B should be created"
2446        );
2447        drop(created);
2448
2449        // Parent should be marked blocked
2450        let updates = backend.status_updates.lock().unwrap();
2451        assert!(
2452            updates
2453                .iter()
2454                .any(|(id, s)| id == "99" && *s == Status::Blocked),
2455            "parent should be blocked"
2456        );
2457        drop(updates);
2458
2459        // A comment summarising the delegations should be posted
2460        let comments = backend.comments.lock().unwrap();
2461        assert!(
2462            !comments.is_empty(),
2463            "a delegation summary comment should be posted"
2464        );
2465        let comment_body = &comments[0].1;
2466        assert!(
2467            comment_body.contains("Subtask A"),
2468            "summary should mention Subtask A"
2469        );
2470        assert!(
2471            comment_body.contains("Subtask B"),
2472            "summary should mention Subtask B"
2473        );
2474
2475        if let Some(old) = old_orch_home {
2476            std::env::set_var("ORCH_HOME", old);
2477        } else {
2478            std::env::remove_var("ORCH_HOME");
2479        }
2480    }
2481
2482    #[allow(clippy::await_holding_lock)]
2483    #[tokio::test]
2484    async fn process_delegations_single_subtask() {
2485        let _guard = ENV_LOCK.lock().unwrap();
2486        let temp_home = TempDir::new().unwrap();
2487        let orch_home = temp_home.path().join(".orch");
2488        tokio::fs::create_dir_all(&orch_home).await.unwrap();
2489        let old_orch_home = std::env::var("ORCH_HOME").ok();
2490        std::env::set_var("ORCH_HOME", &orch_home);
2491
2492        let runner = TaskRunner::new("owner/repo".to_string());
2493        let parent = make_task("101");
2494        let backend = Arc::new(TrackingBackend::new());
2495        let backend_dyn: Arc<dyn crate::backends::ExternalBackend> = backend.clone();
2496
2497        let delegations = vec![crate::parser::Delegation {
2498            title: "Only Child".to_string(),
2499            body: "Do only child".to_string(),
2500            labels: vec!["feature".to_string()],
2501        }];
2502
2503        runner
2504            .process_delegations(&parent, &delegations, &backend_dyn)
2505            .await
2506            .unwrap();
2507
2508        let created = backend.sub_tasks_created.lock().unwrap();
2509        assert_eq!(created.len(), 1);
2510        assert_eq!(created[0].0, "Only Child");
2511        drop(created);
2512
2513        let updates = backend.status_updates.lock().unwrap();
2514        assert!(
2515            updates
2516                .iter()
2517                .any(|(id, s)| id == "101" && *s == Status::Blocked),
2518            "parent should be blocked"
2519        );
2520        drop(updates);
2521
2522        let comments = backend.comments.lock().unwrap();
2523        let body = &comments[0].1;
2524        assert!(
2525            body.contains("1 subtask"),
2526            "comment should count one subtask"
2527        );
2528
2529        if let Some(old) = old_orch_home {
2530            std::env::set_var("ORCH_HOME", old);
2531        } else {
2532            std::env::remove_var("ORCH_HOME");
2533        }
2534    }
2535
2536    // ── weight signal logic ───────────────────────────────────────────────────
2537
2538    /// Test helper that mirrors the weight signal logic from `run_with_context`.
2539    /// Reroutes only emit `RateLimited` when the error type is genuinely "rate_limit".
2540    fn weight_signal_for(status: &str, error_type: &str, agent: &str) -> WeightSignal {
2541        let is_rerouted = status == "new" || status == "routed";
2542        let is_rate_limit_error = error_type == "rate_limit";
2543        if is_rerouted {
2544            if is_rate_limit_error {
2545                WeightSignal::RateLimited {
2546                    agent: agent.to_string(),
2547                }
2548            } else {
2549                WeightSignal::Rerouted
2550            }
2551        } else if status == "done"
2552            || status == "needs_review"
2553            || status == "in_progress"
2554            || status == "in_review"
2555        {
2556            WeightSignal::Success {
2557                agent: agent.to_string(),
2558            }
2559        } else if status == "blocked" {
2560            WeightSignal::Blocked
2561        } else {
2562            WeightSignal::None
2563        }
2564    }
2565
2566    #[test]
2567    fn weight_signal_success_includes_needs_review() {
2568        // The normal happy-path for code tasks: agent reports done, has a PR,
2569        // so handle_success returns "needs_review". This must map to Success.
2570        let signal = weight_signal_for("needs_review", "success", "claude");
2571        assert!(
2572            matches!(signal, WeightSignal::Success { agent } if agent == "claude"),
2573            "needs_review should produce WeightSignal::Success"
2574        );
2575    }
2576
2577    #[test]
2578    fn weight_signal_success_for_all_done_statuses() {
2579        for status in ["done", "needs_review", "in_progress", "in_review"] {
2580            let signal = weight_signal_for(status, "success", "codex");
2581            assert!(
2582                matches!(signal, WeightSignal::Success { agent } if agent == "codex"),
2583                "{status} should produce WeightSignal::Success"
2584            );
2585        }
2586    }
2587
2588    #[test]
2589    fn weight_signal_rate_limited_for_reroute_with_rate_limit_error() {
2590        // Reroute due to rate limit should emit RateLimited.
2591        let signal = weight_signal_for("new", "rate_limit", "claude");
2592        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
2593    }
2594
2595    #[test]
2596    fn weight_signal_rate_limited_for_routed_status_with_rate_limit_error() {
2597        let signal = weight_signal_for("routed", "rate_limit", "claude");
2598        assert!(matches!(signal, WeightSignal::RateLimited { agent } if agent == "claude"));
2599    }
2600
2601    #[test]
2602    fn weight_signal_rerouted_for_non_rate_limit_reroute() {
2603        // Reroute due to silence detection, timeout, or other non-rate-limit errors
2604        // should emit Rerouted (not RateLimited and not None).
2605        // Rerouted maps to "new" in tick so the task stays retryable and is never
2606        // converted to needs_review → done via the no-code review shortcut.
2607        for error_type in ["timeout", "silence detected", "parse_error", "failed"] {
2608            for status in ["new", "routed"] {
2609                let signal = weight_signal_for(status, error_type, "claude");
2610                assert!(
2611                    matches!(signal, WeightSignal::Rerouted),
2612                    "{status} with error_type={error_type} should produce WeightSignal::Rerouted, not RateLimited or None"
2613                );
2614            }
2615        }
2616    }
2617
2618    #[test]
2619    fn silence_reset_rerouted_never_becomes_needs_review() {
2620        // Regression test for: silence detection sets task to new, runner returns
2621        // WeightSignal::Rerouted, tick must map that to "new" — not "needs_review".
2622        // If it mapped to needs_review the no-code shortcut would close the task as
2623        // done with no successful agent run in the audit trail.
2624        let signal = weight_signal_for("new", "silence detected", "claude");
2625        assert!(
2626            matches!(signal, WeightSignal::Rerouted),
2627            "silence-reset reroute must produce Rerouted, never None/needs_review"
2628        );
2629        // Verify tick mapping: Rerouted => "new" (not needs_review or done)
2630        let display_status = match &signal {
2631            WeightSignal::Rerouted => "new",
2632            WeightSignal::None => "needs_review", // the old (buggy) path
2633            _ => "other",
2634        };
2635        assert_eq!(
2636            display_status, "new",
2637            "Rerouted must map to 'new' in tick, not 'needs_review'"
2638        );
2639    }
2640
2641    #[test]
2642    fn weight_signal_blocked_status() {
2643        let signal = weight_signal_for("blocked", "success", "claude");
2644        assert!(matches!(signal, WeightSignal::Blocked));
2645    }
2646
2647    // ── parse_success_output token extraction ────────────────────────────────
2648
2649    /// Mock runner that always returns InvalidResponse with truncated text,
2650    /// simulating what minimax/codex do when they can't parse structured JSON.
2651    struct FailingMockRunner;
2652
2653    impl agents::AgentRunner for FailingMockRunner {
2654        #[cfg(test)]
2655        fn name(&self) -> &str {
2656            "opencode"
2657        }
2658
2659        fn stream_events(&self, _event: &serde_json::Value) -> Vec<agents::StreamEvent> {
2660            vec![]
2661        }
2662
2663        fn build_command(
2664            &self,
2665            _model: Option<&str>,
2666            _timeout_cmd: &str,
2667            _sys_file: &str,
2668            _msg_file: &str,
2669            _permissions: &agents::PermissionRules,
2670        ) -> String {
2671            String::new()
2672        }
2673
2674        fn parse_response(&self, raw: &str) -> Result<agents::ParsedResponse, agents::AgentError> {
2675            // Simulate what minimax does: truncate to 300 chars
2676            Err(agents::AgentError::InvalidResponse {
2677                raw: raw.chars().take(30).collect(),
2678            })
2679        }
2680
2681        fn extract_text(&self, raw: &str) -> Result<String, agents::AgentError> {
2682            Err(agents::AgentError::InvalidResponse {
2683                raw: raw.to_string(),
2684            })
2685        }
2686
2687        fn classify_error(
2688            &self,
2689            exit_code: i32,
2690            _stdout: &str,
2691            _stderr: &str,
2692        ) -> agents::AgentError {
2693            agents::AgentError::Unknown {
2694                exit_code,
2695                message: "mock error".into(),
2696            }
2697        }
2698
2699        fn router_command(
2700            &self,
2701            _prompt: &str,
2702            _model: Option<&str>,
2703        ) -> anyhow::Result<tokio::process::Command> {
2704            anyhow::bail!("not implemented for mock")
2705        }
2706    }
2707
2708    #[test]
2709    fn parse_success_output_extracts_tokens_from_raw_stdout_not_error_raw() {
2710        // OpenCode NDJSON output with token metadata in step_finish
2711        let ndjson = r#"{"type":"text","timestamp":1,"part":{"type":"text","text":"Here is my response: {\"status\":\"done\",\"summary\":\"fixed\"}}"}}
2712{"type":"step_finish","timestamp":2,"part":{"type":"step-finish","reason":"stop","cost":0,"tokens":{"total":95177,"input":94920,"output":257}}}"#;
2713
2714        let runner = FailingMockRunner;
2715        let result = parse_success_output("1550", "opencode", &runner, ndjson);
2716
2717        // Should succeed (synthesized from text) with tokens preserved
2718        let parsed = result.expect("should synthesize response from text");
2719        assert_eq!(parsed.input_tokens, Some(94920));
2720        assert_eq!(parsed.output_tokens, Some(257));
2721    }
2722
2723    #[test]
2724    fn parse_success_output_no_tokens_when_ndjson_has_none() {
2725        // Plain text with an explicit completion signal but no token metadata.
2726        // Uses "No open positions" which is in the explicit_done list.
2727        let text = "No open positions, no trade executions, no conditions change.";
2728
2729        let runner = FailingMockRunner;
2730        let result = parse_success_output("1551", "opencode", &runner, text);
2731
2732        // Should synthesize a "done" response, but without tokens (no NDJSON envelope).
2733        let parsed = result.expect("should synthesize response from explicit-done text");
2734        assert_eq!(parsed.input_tokens, None);
2735        assert_eq!(parsed.output_tokens, None);
2736    }
2737
2738    #[test]
2739    fn parse_success_output_uses_agent_result_directly() {
2740        // Here we test that if find_agent_result succeeds and returns valid JSON, we use it directly
2741        // WITHOUT calling agent_runner.parse_response at all.
2742        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"result":"{\"status\":\"done\",\"summary\":\"my summary\",\"accomplished\":[],\"remaining\":[],\"files\":[]}"}"#;
2743
2744        // FailingMockRunner always returns InvalidResponse, so if we called it, we'd synthesize "done" instead of getting "my summary".
2745        let runner = FailingMockRunner;
2746        let result = parse_success_output("999", "claude", &runner, ndjson);
2747
2748        let parsed = result.expect("should parse successfully via find_agent_result");
2749        assert_eq!(parsed.response.status, "done");
2750        assert_eq!(parsed.response.summary, "my summary");
2751    }
2752
2753    #[test]
2754    fn parse_success_output_treats_agent_result_is_error_as_authoritative() {
2755        // Here we test that if find_agent_result returns is_error=true, we return AgentFailed
2756        // instead of falling back to synthesis or agent_runner.parse_response.
2757        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"I failed because of reasons"}"#;
2758
2759        let runner = FailingMockRunner;
2760        let result = parse_success_output("999", "claude", &runner, ndjson);
2761
2762        let err = result.expect_err("should return error");
2763        match err {
2764            agents::AgentError::AgentFailed { message } => {
2765                assert_eq!(message, "I failed because of reasons");
2766            }
2767            _ => panic!("expected AgentFailed, got {err:?}"),
2768        }
2769    }
2770
2771    #[test]
2772    fn parse_success_output_classifies_model_unavailable_from_is_error_result() {
2773        // Regression test for issue #3607: the is_error=true NDJSON path
2774        // (used by codex's find_codex_result, which stringifies its own
2775        // classified AgentError into result_text) must re-classify a
2776        // model-not-found message as ModelUnavailable so it gets
2777        // model-scoped cooldown treatment instead of degrading to a
2778        // generic AgentFailed, which triggers an agent-wide cooldown.
2779        let ndjson = r#"{"type":"result","subtype":"success","is_error":true,"result":"model unavailable (gpt-5.5): Reconnecting... 2/5 (unexpected status 404 Not Found: The model `gpt-5.5` does not exist or you do not have access to it.)"}"#;
2780
2781        let runner = FailingMockRunner;
2782        let result = parse_success_output("999", "claude", &runner, ndjson);
2783
2784        let err = result.expect_err("should return error");
2785        match err {
2786            agents::AgentError::ModelUnavailable { model, .. } => {
2787                assert_eq!(model, "gpt-5.5");
2788            }
2789            _ => panic!("expected ModelUnavailable, got {err:?}"),
2790        }
2791    }
2792
2793    #[test]
2794    fn nonzero_exit_with_ndjson_success_is_parsed_as_success() {
2795        // Simulate an agent that exits with non-zero code but emits NDJSON result
2796        // with is_error=false and a JSON result payload. The runner should accept
2797        // this as success because the NDJSON result is authoritative.
2798        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1234,"result":"{\"status\":\"done\",\"summary\":\"All good\"}","usage":{"input_tokens":10,"output_tokens":5}}"#;
2799
2800        // Use a runner that would normally fail parsing to ensure we're using
2801        // the find_agent_result path. FailingMockRunner.parse_response returns
2802        // InvalidResponse, but parse_success_output should short-circuit.
2803        let runner = FailingMockRunner;
2804
2805        // Construct a fake SessionOutput
2806        let session_output = session::SessionOutput {
2807            exit_code: 1,
2808            raw_stdout: ndjson.to_string(),
2809            raw_stderr: String::new(),
2810            elapsed_secs: Some(1),
2811        };
2812
2813        // Call the helper we added via the task runner logic by reusing the
2814        // parse_success_output directly since it encapsulates the same behavior.
2815        let parsed =
2816            parse_success_output("task-ndjson", "claude", &runner, &session_output.raw_stdout)
2817                .expect("should parse NDJSON success");
2818        assert_eq!(parsed.response.status, "done");
2819        assert_eq!(parsed.response.summary, "All good");
2820        assert_eq!(parsed.input_tokens, Some(10));
2821        assert_eq!(parsed.output_tokens, Some(5));
2822    }
2823
2824    // ── agent label update after reroute ─────────────────────────────────────
2825
2826    /// Backend that wraps TrackingBackend but can be configured to fail label ops.
2827    struct LabelFailingBackend {
2828        inner: TrackingBackend,
2829        /// If set, set_labels returns this error.
2830        set_labels_err: Option<String>,
2831    }
2832
2833    impl LabelFailingBackend {
2834        fn new() -> Self {
2835            Self {
2836                inner: TrackingBackend::new(),
2837                set_labels_err: None,
2838            }
2839        }
2840        fn with_set_labels_err(mut self, err: impl Into<String>) -> Self {
2841            self.set_labels_err = Some(err.into());
2842            self
2843        }
2844    }
2845
2846    impl Default for LabelFailingBackend {
2847        fn default() -> Self {
2848            Self::new()
2849        }
2850    }
2851
2852    #[async_trait]
2853    impl crate::backends::ExternalBackend for LabelFailingBackend {
2854        fn name(&self) -> &str {
2855            "label_failing"
2856        }
2857        async fn create_task(
2858            &self,
2859            _t: &str,
2860            _b: &str,
2861            _l: &[String],
2862        ) -> anyhow::Result<ExternalId> {
2863            Ok(ExternalId("new".to_string()))
2864        }
2865        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
2866            self.inner.get_task(id).await
2867        }
2868        async fn list_by_status(&self, _s: Status) -> anyhow::Result<Vec<ExternalTask>> {
2869            Ok(vec![])
2870        }
2871        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2872            Ok(vec![])
2873        }
2874        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2875            self.inner.post_comment(id, body).await
2876        }
2877        async fn set_labels(&self, id: &ExternalId, labels: &[String]) -> anyhow::Result<()> {
2878            if let Some(ref err) = self.set_labels_err {
2879                return Err(anyhow::anyhow!("{}", err));
2880            }
2881            self.inner.set_labels(id, labels).await
2882        }
2883        async fn remove_label(&self, id: &ExternalId, label: &str) -> anyhow::Result<()> {
2884            self.inner.remove_label(id, label).await
2885        }
2886        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2887            Ok(vec![])
2888        }
2889        async fn create_sub_task(
2890            &self,
2891            parent: &ExternalId,
2892            title: &str,
2893            body: &str,
2894            labels: &[String],
2895        ) -> anyhow::Result<ExternalId> {
2896            self.inner
2897                .create_sub_task(parent, title, body, labels)
2898                .await
2899        }
2900        async fn ensure_status_label(&self, label: &str) -> anyhow::Result<()> {
2901            self.inner.ensure_status_label(label).await
2902        }
2903        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
2904            self.inner.update_status(id, status).await
2905        }
2906        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
2907            self.inner.get_authenticated_user().await
2908        }
2909        async fn get_mentions(&self, _s: &str) -> anyhow::Result<Vec<Mention>> {
2910            Ok(vec![])
2911        }
2912        async fn health_check(&self) -> anyhow::Result<()> {
2913            self.inner.health_check().await
2914        }
2915    }
2916
2917    /// `update_agent_label_after_reroute` returns `false` when `set_labels` fails.
2918    ///
2919    /// This tests the extracted helper directly, bypassing the full `run_with_context`
2920    /// path (which would fail earlier on worktree setup in a unit-test environment).
2921    #[tokio::test]
2922    async fn update_agent_label_returns_false_on_set_labels_failure() {
2923        let task = ExternalTask {
2924            id: ExternalId("test-reroute-1".to_string()),
2925            title: "Test".to_string(),
2926            body: "body".to_string(),
2927            state: "open".to_string(),
2928            labels: vec![],
2929            author: "bot".to_string(),
2930            created_at: "2026-01-01T00:00:00Z".to_string(),
2931            updated_at: "2026-01-01T00:00:00Z".to_string(),
2932            url: "".to_string(),
2933        };
2934        let backend: Arc<dyn crate::backends::ExternalBackend> =
2935            Arc::new(LabelFailingBackend::new().with_set_labels_err("network error"));
2936
2937        let result =
2938            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
2939                .await;
2940
2941        assert!(
2942            !result,
2943            "update_agent_label_after_reroute should return false when set_labels fails"
2944        );
2945    }
2946
2947    /// `update_agent_label_after_reroute` returns `true` when `set_labels` succeeds.
2948    #[tokio::test]
2949    async fn update_agent_label_returns_true_on_success() {
2950        let task = ExternalTask {
2951            id: ExternalId("test-reroute-2".to_string()),
2952            title: "Test".to_string(),
2953            body: "body".to_string(),
2954            state: "open".to_string(),
2955            labels: vec![],
2956            author: "bot".to_string(),
2957            created_at: "2026-01-01T00:00:00Z".to_string(),
2958            updated_at: "2026-01-01T00:00:00Z".to_string(),
2959            url: "".to_string(),
2960        };
2961        // LabelFailingBackend with no injected error delegates to TrackingBackend (always Ok).
2962        let backend: Arc<dyn crate::backends::ExternalBackend> =
2963            Arc::new(LabelFailingBackend::new());
2964
2965        let result =
2966            update_agent_label_after_reroute("owner/repo", &task, &backend, "claude", "codex")
2967                .await;
2968
2969        // ensure_label is skipped in #[cfg(test)], set_labels succeeds via mock.
2970        assert!(
2971            result,
2972            "update_agent_label_after_reroute should return true when set_labels succeeds"
2973        );
2974    }
2975
2976    // ── NDJSON envelope fallback (issue #3078) ────────────────────────────────
2977
2978    /// Pattern A: agent emits domain JSON as result (no `status` field).
2979    /// NDJSON envelope says is_error=false, subtype=success.
2980    /// Was previously returning InvalidResponse → task re-ran.
2981    #[test]
2982    fn parse_success_output_ndjson_envelope_fallback_domain_json_result() {
2983        let domain_result = r#"{"file_path":"md/journal/2026-05-08-morning-briefing.md","summary":"Morning briefing filed","health":{"status":"ok"},"priorities":[]}"#;
2984        let ndjson = format!(
2985            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{domain_result:?},"usage":{{"input_tokens":3000,"output_tokens":400}}}}"#
2986        );
2987
2988        let runner = agents::get_runner("glm");
2989        let parsed = parse_success_output("149200", "glm", &*runner, &ndjson)
2990            .expect("should synthesize done from NDJSON envelope when result is domain JSON");
2991
2992        assert_eq!(parsed.response.status, "done");
2993        assert!(
2994            parsed.response.summary.contains("file_path") || !parsed.response.summary.is_empty(),
2995            "summary should be populated from domain JSON result text"
2996        );
2997        assert_eq!(parsed.input_tokens, Some(3000));
2998        assert_eq!(parsed.output_tokens, Some(400));
2999    }
3000
3001    /// Pattern B: agent emits plain prose as result, no completion keywords.
3002    /// NDJSON envelope says is_error=false, subtype=success.
3003    /// Was previously returning InvalidResponse → task re-ran.
3004    #[test]
3005    fn parse_success_output_ndjson_envelope_fallback_plain_prose_result() {
3006        let prose = "PR #1008 now has zero diff vs main. The reverted ledger changes are gone and the branch is clean.";
3007        let ndjson = format!(
3008            r#"{{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":{prose:?},"usage":{{"input_tokens":2500,"output_tokens":300}}}}"#
3009        );
3010
3011        let runner = agents::get_runner("claude");
3012        let parsed = parse_success_output("149164", "claude", &*runner, &ndjson)
3013            .expect("should synthesize done from NDJSON envelope when result is plain prose");
3014
3015        assert_eq!(parsed.response.status, "done");
3016        assert_eq!(parsed.response.summary, prose);
3017        assert_eq!(parsed.input_tokens, Some(2500));
3018        assert_eq!(parsed.output_tokens, Some(300));
3019    }
3020
3021    /// Verify that is_error=true in NDJSON envelope still returns AgentFailed,
3022    /// not "done" — the envelope fallback must not override explicit errors.
3023    #[test]
3024    fn parse_success_output_ndjson_envelope_fallback_skipped_when_is_error_true() {
3025        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"result":"Something went wrong with no status"}"#;
3026
3027        let runner = agents::get_runner("claude");
3028        let err = parse_success_output("149999", "claude", &*runner, ndjson)
3029            .expect_err("is_error=true should return AgentFailed");
3030
3031        assert!(matches!(err, agents::AgentError::AgentFailed { .. }));
3032    }
3033
3034    // ── terminal_reason:completed + exit 1 detection (issue #3087) ─────────────
3035
3036    /// kimi/minimax sometimes emit valid NDJSON with terminal_reason=completed
3037    /// but exit 1. In that case parse_success_output may fail (e.g. empty result
3038    /// text that synthesize can't parse), then parse_session_output would
3039    /// normally fall back to classify_error — grabbing a garbled JSON tail and
3040    /// producing a spurious Unknown/Auth error. The fix intercepts this and
3041    /// returns InvalidResponse (soft error, re-route) instead.
3042    #[test]
3043    fn parse_session_output_exit1_with_terminal_reason_completed_returns_invalid_response() {
3044        // Empty result text causes synthesize_response_from_text to return None
3045        // (empty/whitespace-only input), so parse_success_output returns
3046        // InvalidResponse. With the fix, parse_session_output intercepts the
3047        // "terminal_reason":"completed" signal before classify_error and
3048        // returns InvalidResponse (re-route) instead of a garbled error.
3049        let ndjson = r#"{"type":"result","subtype":"success","is_error":false,"terminal_reason":"completed","result":"","usage":{"input_tokens":1000,"output_tokens":100}}"#;
3050        let session = session::SessionOutput {
3051            exit_code: 1,
3052            raw_stdout: ndjson.to_string(),
3053            raw_stderr: String::new(),
3054            elapsed_secs: Some(30),
3055        };
3056
3057        let runner = agents::get_runner("claude");
3058        let result = parse_session_output("149298", "claude", &*runner, &session);
3059
3060        let err = result.expect_err("should be an error, not Ok");
3061        assert!(
3062            matches!(err, agents::AgentError::InvalidResponse { .. }),
3063            "should be InvalidResponse, got {err:?}"
3064        );
3065    }
3066
3067    // ── GLM cost-telemetry exit-1 detection (issue #3094) ──────────────────────
3068
3069    /// GLM exits with code 1 but stdout is only cost telemetry JSON (contains
3070    /// "costUSD":). This must return InvalidResponse (soft re-route) instead of
3071    /// a hard Failed classification with spurious agent cooldown.
3072    #[test]
3073    fn parse_session_output_exit1_with_cost_telemetry_returns_invalid_response() {
3074        // GLM emits cost/usage JSON on clean exit; exit code 1 is a false failure.
3075        let telemetry = r#"{"type":"result","subtype":"success","is_error":false,"result":"","costUSD":0.042,"usage":{"input_tokens":5000,"output_tokens":800}}"#;
3076        let session = session::SessionOutput {
3077            exit_code: 1,
3078            raw_stdout: telemetry.to_string(),
3079            raw_stderr: String::new(),
3080            elapsed_secs: Some(120),
3081        };
3082
3083        let runner = agents::get_runner("glm");
3084        let result = parse_session_output("149347", "glm", &*runner, &session);
3085
3086        let err = result.expect_err("should be an error, not Ok");
3087        assert!(
3088            matches!(err, agents::AgentError::InvalidResponse { .. }),
3089            "glm cost-telemetry exit-1 should return InvalidResponse, got {err:?}"
3090        );
3091    }
3092
3093    /// GLM with costUSD AND is_error:true — must propagate AgentFailed, not
3094    /// silently swallow the error via the cost-telemetry guard.
3095    #[test]
3096    fn parse_session_output_exit1_cost_telemetry_with_is_error_true_propagates_failure() {
3097        let telemetry = r#"{"type":"result","subtype":"error","is_error":true,"result":"tool failed","costUSD":0.01,"usage":{"input_tokens":100,"output_tokens":10}}"#;
3098        let session = session::SessionOutput {
3099            exit_code: 1,
3100            raw_stdout: telemetry.to_string(),
3101            raw_stderr: String::new(),
3102            elapsed_secs: Some(15),
3103        };
3104
3105        let runner = agents::get_runner("glm");
3106        let result = parse_session_output("149348", "glm", &*runner, &session);
3107
3108        let err = result.expect_err("should be an error");
3109        assert!(
3110            matches!(err, agents::AgentError::AgentFailed { .. }),
3111            "is_error=true with costUSD must still return AgentFailed, got {err:?}"
3112        );
3113    }
3114
3115    /// terminal_reason=completed must NOT override actual error conditions.
3116    /// When is_error=true in the NDJSON envelope, AgentFailed must be returned
3117    /// even if terminal_reason=completed is also present. Only kimi/minimax
3118    /// agents use find_claude_result which extracts is_error, so use "kimi"
3119    /// as agent_name.
3120    #[test]
3121    fn parse_session_output_exit1_preserves_error_when_is_error_true() {
3122        let ndjson = r#"{"type":"result","subtype":"error","is_error":true,"terminal_reason":"completed","result":"failed"}"#;
3123        let session = session::SessionOutput {
3124            exit_code: 1,
3125            raw_stdout: ndjson.to_string(),
3126            raw_stderr: String::new(),
3127            elapsed_secs: Some(30),
3128        };
3129
3130        let runner = agents::get_runner("kimi");
3131        let result = parse_session_output("149299", "kimi", &*runner, &session);
3132
3133        let err = result.expect_err("should be an error");
3134        assert!(
3135            matches!(err, agents::AgentError::AgentFailed { .. }),
3136            "is_error=true should return AgentFailed, got {err:?}"
3137        );
3138    }
3139
3140    /// RateLimit with terminal_reason=completed must be propagated as RateLimit,
3141    /// not downgraded to InvalidResponse. This covers claude session-limit errors
3142    /// that include "terminal_reason":"completed" in their NDJSON envelope — the
3143    /// terminal_reason guard was designed for kimi/minimax/glm false-positive
3144    /// exit codes but was intercepting classified errors like RateLimit.
3145    #[test]
3146    fn parse_session_output_exit1_with_rate_limit_and_terminal_reason_completed() {
3147        let ndjson = r#"{"type":"result","is_error":true,"terminal_reason":"completed","result":"You've hit your session limit · resets 2:20pm (America/Sao_Paulo)","usage":{"input_tokens":0,"output_tokens":0}}"#;
3148        let session = session::SessionOutput {
3149            exit_code: 1,
3150            raw_stdout: ndjson.to_string(),
3151            raw_stderr: String::new(),
3152            elapsed_secs: Some(120),
3153        };
3154
3155        let runner = agents::get_runner("claude");
3156        let result = parse_session_output("152895", "claude", &*runner, &session);
3157
3158        let err = result.expect_err("rate-limit should be an error");
3159        assert!(
3160            matches!(err, agents::AgentError::RateLimit { .. }),
3161            "session limit should return RateLimit, not downgraded, got {err:?}"
3162        );
3163    }
3164}