Skip to main content

orch/engine/
tick.rs

1//! Core tick loop phases.
2//!
3//! The engine ticks every ~10 seconds. Each tick runs six phases:
4//! 1. Poll tmux for finished sessions
5//! 2. Recover stuck in-progress tasks
6//! 3. Route and dispatch:
7//!    - 3a. Route new tasks to agents
8//!    - 3b. Dispatch routed tasks (spawn agents)
9//! 4. Unblock parents whose children are all done
10//! 5. Run cron job scheduler
11//!
12//! [`tick_all`] runs them for every project. Phases 1-2 and 3b-5 run for all projects at
13//! once; only routing is serial, because it holds the router write lock. Store-wide work
14//! (`tick_global_sweeps`, `reconcile_dependencies`) runs once per tick, not once per project.
15
16use super::{EngineConfig, ProjectEngine};
17use crate::backends::github::GitHubBackend;
18use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
19use crate::channels::capture::CaptureService;
20use crate::channels::transport::Transport;
21use crate::cmd_cache;
22use crate::config;
23use crate::engine;
24use crate::engine::cooldown::{
25    SILENCE_AGENT_COOLDOWN_SECS, SILENCE_EXTENDED_COOLDOWN_SECS, github_circuit_remaining_secs,
26    is_agent_in_cooldown, is_github_circuit_open, is_model_in_cooldown,
27    record_agent_failure_with_message, record_silence_detection, set_agent_cooldown,
28    set_model_cooldown,
29};
30use crate::engine::dependencies;
31use crate::engine::dispatch_guard::DispatchGuard;
32use crate::engine::jobs;
33use crate::engine::priority;
34use crate::engine::priority::Priority;
35use crate::engine::router::config as router_config;
36use crate::engine::router::{
37    AllAgentsCooledError, RouteResultError, Router, RouterConfig, get_route_result,
38};
39use crate::engine::runner::response;
40use crate::engine::runner::{NoProjectDir, TaskRunner, WeightSignal};
41use crate::engine::sync;
42use crate::engine::tasks;
43use crate::engine::tasks::{TaskManager, is_internal_id};
44use crate::github::http;
45use crate::repo_context::REPO_CONTEXT;
46use crate::store;
47use crate::store::StoreRoute;
48use crate::store::Task;
49use crate::store::TaskStatus;
50use crate::store::TaskStatus as DbStatus;
51use crate::store::TaskStore;
52use crate::store::WaitingOnDependency;
53use crate::store::{store_set_by_id, store_touch_updated_at};
54use crate::tmux::TmuxManager;
55use dashmap::mapref::entry::Entry;
56use dashmap::{DashMap, DashSet};
57use futures::future;
58use std::collections::HashMap;
59use std::collections::HashSet;
60use std::path::Path;
61use std::sync::Arc;
62use std::sync::OnceLock;
63use std::time::Duration;
64use std::time::{Instant, SystemTime};
65use tokio::sync::RwLock;
66use tokio::sync::{Semaphore, mpsc};
67use tokio::time;
68
69/// Per outer engine-loop quota for expensive routing calls.
70///
71/// Routing runs once per active project. Without a quota shared by all of them,
72/// `router.max_tasks_per_tick = 1` still permits one slow LLM routing call per
73/// project, multiplying 45s router timeouts into multi-minute watchdog stalls.
74pub(crate) struct RoutingTickQuota {
75    remaining: usize,
76}
77
78impl RoutingTickQuota {
79    pub(crate) fn new(max: usize) -> Self {
80        Self { remaining: max }
81    }
82
83    fn is_exhausted(&self) -> bool {
84        self.remaining == 0
85    }
86
87    fn remaining(&self) -> usize {
88        self.remaining
89    }
90
91    fn consume_one(&mut self) -> bool {
92        if self.remaining == 0 {
93            return false;
94        }
95        self.remaining -= 1;
96        true
97    }
98}
99
100/// Cleanup stale tmux sessions from previous engine runs.
101/// Called once from `serve()` before the main loop, never from a tick
102pub(crate) async fn startup_cleanup(tmux: &TmuxManager) {
103    let start_time = SystemTime::now()
104        .duration_since(SystemTime::UNIX_EPOCH)
105        .unwrap_or_default()
106        .as_secs();
107    tracing::info!(
108        start_time,
109        "running startup cleanup for stale tmux sessions"
110    );
111
112    match tmux.kill_stale_sessions(start_time).await {
113        Ok(killed) => {
114            if killed > 0 {
115                tracing::info!(killed, "startup cleanup killed stale sessions");
116            } else {
117                tracing::debug!("no stale tmux sessions found on startup");
118            }
119        }
120        Err(e) => {
121            tracing::warn!(error = %e, "failed to run startup cleanup");
122        }
123    }
124}
125
126async fn recover_routed_blocked_dispatch(
127    backend: &Arc<dyn ExternalBackend>,
128    store: &Arc<TaskStore>,
129    task_manager: &Arc<TaskManager>,
130    repo: &str,
131    task: &ExternalTask,
132    timing: &StuckTaskTiming,
133) {
134    tracing::warn!(
135        task_id = task.id.0,
136        age_mins = timing.age.num_minutes(),
137        threshold_mins = timing.threshold / 60,
138        "recovering routed task: dispatch blocked by stale tmux session → new"
139    );
140
141    let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
142        Ok(Some(id)) => Some(id),
143        Ok(None) => {
144            tracing::warn!(
145                task_id = %task.id.0,
146                repo,
147                "resolve_task_id returned None during routed-task recovery"
148            );
149            None
150        }
151        Err(e) => {
152            tracing::error!(
153                task_id = %task.id.0,
154                repo,
155                error = %e,
156                "resolve_task_id failed during routed-task recovery"
157            );
158            None
159        }
160    };
161
162    if let Some(store_id) = resolved_store_id {
163        if let Err(e) = store_set_by_id(
164            Some(store),
165            store_id,
166            &[
167                ("agent", serde_json::Value::Null),
168                ("model", serde_json::Value::Null),
169                ("route_attempts", serde_json::json!(0)),
170            ],
171        )
172        .await
173        {
174            tracing::warn!(
175                task_id = task.id.0,
176                ?e,
177                "failed to clear routing fields for routed stuck task — will retry next tick"
178            );
179            return;
180        }
181        finalize_open_runs_for_recovery(
182            store,
183            store_id,
184            "stuck-task recovery: stale session blocking dispatch killed",
185        )
186        .await;
187    }
188
189    if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
190        tracing::warn!(
191            task_id = task.id.0,
192            ?e,
193            "failed to reset routed stuck task status"
194        );
195        return;
196    }
197
198    store::store_log_activity(
199        &Some(Arc::clone(store)),
200        repo,
201        &task.id.0,
202        "rerouted",
203        Some("routed"),
204        Some("new"),
205        None::<&str>,
206        None::<&str>,
207        Some(&serde_json::json!({
208            "failure_reason": "routed_blocked_by_existing_session",
209            "age_minutes": timing.age.num_minutes(),
210            "had_session": true,
211        })),
212    )
213    .await;
214
215    if !is_internal_id(&task.id.0) {
216        let backend_clone = Arc::clone(backend);
217        let task_id_clone = task.id.clone();
218        let body = format!(
219            "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
220            chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
221            timing.age.num_minutes(),
222            engine::orch_footer()
223        );
224        tokio::spawn(async move {
225            if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
226                tracing::warn!(
227                    task_id = task_id_clone.0,
228                    ?e,
229                    "failed to post routed-task recovery comment"
230                );
231            }
232        });
233    }
234}
235
236/// Phase 1 of tick: poll tmux for finished sessions and clean them up.
237pub(crate) async fn tick_check_session_completions(
238    tmux: &Arc<TmuxManager>,
239    repo: &str,
240    capture: &Arc<CaptureService>,
241    store: &Arc<TaskStore>,
242) -> anyhow::Result<()> {
243    let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
244    let session_snapshot = tmux.snapshot().await;
245    // Derive the short project name from repo (owner/repo -> repo), matching
246    // TmuxManager::session_name's derivation exactly (including the ".git" strip)
247    // so a repo configured with a ".git" suffix doesn't silently fail to match
248    // its own live sessions here — leaving stuck-task recovery as the only
249    // (later, coarser-grained) safety net for this repo's session completions.
250    let repo_name = repo
251        .rsplit('/')
252        .next()
253        .unwrap_or(repo)
254        .trim_end_matches(".git");
255    for (session, active) in session_snapshot {
256        // Only handle sessions that belong to this repo/project.
257        if session.project != repo_name {
258            continue;
259        }
260
261        if !active {
262            tracing::info!(
263                session = %session.name,
264                task_id = %session.task_id,
265                "session completed, collecting results"
266            );
267            // Touch the store.updated_at immediately when we observe a completed
268            // session so the stuck-task recovery phase doesn't incorrectly
269            // reclaim this task while the runner is still finishing
270            // post-processing (race: session observed dead → tick reclaim).
271            // Best-effort: no-ops if the task isn't present in the store.
272            //
273            // session.task_id uses hyphens (e.g. "internal-63714") because the
274            // tmux session name sanitizes colons to hyphens. resolve_task_id only
275            // handles "internal:" prefix, so we must convert back before touching.
276            // Review sessions carry a "-review" suffix tracked under the main task.
277            let store_task_id = {
278                let base = session
279                    .task_id
280                    .strip_suffix("-review")
281                    .unwrap_or(&session.task_id);
282                if let Some(n) = base.strip_prefix("internal-") {
283                    format!("internal:{n}")
284                } else {
285                    base.to_string()
286                }
287            };
288            store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
289            // Unregister from capture service using the task id the capture service
290            // was registered under (task id without project prefix).
291            capture.unregister_session(repo, &session.task_id).await;
292            // Kill the actual tmux session name we discovered (do not reconstruct)
293            if let Err(e) = tmux.kill_session(&session.name).await {
294                tracing::debug!(
295                    session = %session.name,
296                    ?e,
297                    "kill_session failed (session may already be gone)"
298                );
299            }
300        }
301    }
302    Ok(())
303}
304
305#[derive(Debug)]
306struct StuckTaskTiming {
307    has_session: bool,
308    age: chrono::Duration,
309    threshold: u64,
310}
311
312fn stuck_task_timing_from_map(
313    tmux: &Arc<TmuxManager>,
314    repo: &str,
315    session_task_id: &str,
316    updated_at: &str,
317    config: &EngineConfig,
318    parse_error_message: &'static str,
319    session_map: &HashMap<String, bool>,
320) -> Option<StuckTaskTiming> {
321    let has_session = session_alive(tmux, repo, session_task_id, session_map);
322    let threshold = if has_session {
323        config.stuck_timeout
324    } else {
325        config.no_session_stuck_timeout
326    };
327
328    let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
329        Ok(dt) => dt.with_timezone(&chrono::Utc),
330        Err(e) => {
331            tracing::warn!(
332                task_id = session_task_id,
333                updated_at,
334                ?e,
335                error_message = parse_error_message,
336                "cannot parse updated_at"
337            );
338            return None;
339        }
340    };
341
342    let raw_age = chrono::Utc::now() - updated;
343    // Host suspend (laptop sleep) freezes the process just like it freezes the agent
344    // running inside it — that frozen time is not evidence of a hang. Discount any
345    // detected suspend/resume gaps that occurred since this task was last touched.
346    let suspended = super::suspend::suspended_duration_since(updated);
347    let age = (raw_age - suspended).max(chrono::Duration::zero());
348    if age.num_seconds() <= i64::try_from(threshold).unwrap_or(i64::MAX) {
349        return None;
350    }
351
352    Some(StuckTaskTiming {
353        has_session,
354        age,
355        threshold,
356    })
357}
358
359fn session_alive(
360    tmux: &TmuxManager,
361    repo: &str,
362    session_task_id: &str,
363    session_map: &HashMap<String, bool>,
364) -> bool {
365    session_map
366        .get(&tmux.session_name(repo, session_task_id))
367        .copied()
368        .unwrap_or(false)
369}
370
371/// The tier whose run timeout the task runs with: the tier of its stored model, which differs from
372/// its own complexity when the model came from an adjacent tier
373fn run_tier(router: &RouterConfig, task: &Task) -> String {
374    match (task.agent.as_deref(), task.model.as_deref()) {
375        (Some(a), Some(m)) if !m.is_empty() => router.tier_of_model(a, m, &task.complexity),
376        _ => task.complexity.clone(),
377    }
378}
379
380/// The router config for this tick, loaded on first use. A full load re-reads the config and
381/// repeats its startup warnings, so one tick shares one load across every live run
382fn tick_router(
383    cell: &OnceLock<RouterConfig>,
384    load: impl FnOnce() -> RouterConfig,
385) -> &RouterConfig {
386    cell.get_or_init(load)
387}
388
389/// Start time and config for the stuck check of an in_progress task. A live session ages from
390/// the status entry, since other writes refresh `updated_at` mid-run, and is not stuck before
391/// its own run timeout ends. A dead session keeps `updated_at`, which the runner touches when it
392/// sees the session end
393async fn in_progress_stuck_clock(
394    store: &TaskStore,
395    repo: &str,
396    task_id: &str,
397    updated_at: &str,
398    has_session: bool,
399    config: &EngineConfig,
400    router: &OnceLock<RouterConfig>,
401) -> (String, EngineConfig) {
402    if !has_session {
403        return (updated_at.to_string(), config.clone());
404    }
405
406    let task = match store.resolve_task_id(repo, task_id).await {
407        Ok(Some(store_id)) => store.get(store_id).await.ok(),
408        _ => None,
409    };
410    let Some(task) = task else {
411        return (updated_at.to_string(), config.clone());
412    };
413
414    let entered_at = match store
415        .status_entered_at(task.id, TaskStatus::InProgress)
416        .await
417    {
418        Ok(Some(ts)) => ts,
419        Ok(None) => updated_at.to_string(),
420        Err(e) => {
421            tracing::warn!(
422                task_id,
423                error = %e,
424                "failed to read in_progress entry, falling back to updated_at"
425            );
426            updated_at.to_string()
427        }
428    };
429    let tier = run_tier(tick_router(router, RouterConfig::from_config), &task);
430    let task_config = EngineConfig {
431        stuck_timeout: config.live_session_stuck_limit(Some(&tier)),
432        ..config.clone()
433    };
434    (entered_at, task_config)
435}
436
437/// When the task last entered in_review, and whether a review decision landed since. Age from the
438/// status entry: `review_open_prs` rewrites `updated_at` on every pass, so it never ages
439async fn in_review_stuck_entry(store: &TaskStore, store_id: i64, fallback: &str) -> (String, bool) {
440    match store.in_review_entry(store_id).await {
441        Ok(Some(entry)) => entry,
442        Ok(None) => (fallback.to_string(), false),
443        Err(e) => {
444            tracing::warn!(
445                store_id,
446                error = %e,
447                "failed to read in_review entry, falling back to updated_at"
448            );
449            (fallback.to_string(), false)
450        }
451    }
452}
453
454/// Finalize any open `task_runs` rows for a task during recovery paths.
455/// Best-effort: logs a warning on failure but does not block recovery.
456async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
457    if let Err(e) = store
458        .finalize_incomplete_runs(store_id, "aborted", reason)
459        .await
460    {
461        tracing::warn!(
462            store_id,
463            error = %e,
464            reason,
465            "failed to finalize incomplete runs during recovery"
466        );
467    }
468}
469
470/// Phase 1b of tick: detect agents that have produced no output since session start
471/// and have exceeded the silence grace period.
472///
473/// Only triggers on complete silence (agent never produced any output). Agents that
474/// produced output then went quiet (e.g. long tool calls) are NOT affected.
475///
476/// On silence: kills the tmux session, applies model-level cooldown, resets the task
477/// to `New` so the router picks a different agent/model.
478pub(crate) async fn tick_detect_silent_agents(
479    tmux: &Arc<TmuxManager>,
480    repo: &str,
481    capture: &Arc<CaptureService>,
482    backend: &Arc<dyn ExternalBackend>,
483    task_manager: &Arc<TaskManager>,
484    config: &EngineConfig,
485    store: &Arc<TaskStore>,
486) -> anyhow::Result<()> {
487    let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
488    let grace = Duration::from_secs(config.silence_grace_period);
489    let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
490
491    for (task_id, session_name, silence_age_secs) in silent_sessions {
492        let use_backend = should_use_backend(&task_id);
493        // Look up agent + model from the store so we can cooldown the right model.
494        let store_task = match store.resolve_task_id(repo, &task_id).await {
495            Ok(Some(store_id)) => match store.get(store_id).await {
496                Ok(task) => Some(task),
497                Err(e) => {
498                    tracing::warn!(
499                        task_id,
500                        repo,
501                        store_id,
502                        operation = "store.get",
503                        error = %e,
504                        "silence detection: db error fetching task — agent/model cooldown will be skipped"
505                    );
506                    None
507                }
508            },
509            Ok(None) => {
510                tracing::debug!(
511                    task_id,
512                    repo,
513                    operation = "resolve_task_id",
514                    "silence detection: task not found in store (may have been cleaned up)"
515                );
516                None
517            }
518            Err(e) => {
519                tracing::warn!(
520                    task_id,
521                    repo,
522                    operation = "resolve_task_id",
523                    error = %e,
524                    "silence detection: db error resolving task id — agent/model cooldown will be skipped"
525                );
526                None
527            }
528        };
529        let agent_name = store_task
530            .as_ref()
531            .and_then(|t| t.agent.clone())
532            .unwrap_or_default();
533        let model_name = store_task
534            .as_ref()
535            .and_then(|t| t.model.clone())
536            .unwrap_or_default();
537
538        tracing::warn!(
539            task_id,
540            agent = %agent_name,
541            model = %model_name,
542            silence_age_secs,
543            grace_secs = config.silence_grace_period,
544            cooldown_secs = config.silence_cooldown,
545            "agent silent since session start — killing session, cooling down model + agent, failing over"
546        );
547
548        // 1. Kill the tmux session
549        if let Err(e) = tmux.kill_session(&session_name).await {
550            tracing::debug!(
551                task_id,
552                ?e,
553                "kill_session failed for silent agent (may already be gone)"
554            );
555        }
556
557        // 2. Unregister from capture
558        capture.unregister_session(repo, &task_id).await;
559
560        // Finalize any open audit run so the task_runs row is not orphaned
561        if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
562            finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
563                .await;
564        }
565
566        let mut extended_note = String::new();
567
568        // 3. Cooldown the specific model (not the whole agent)
569        if !agent_name.is_empty() && !model_name.is_empty() {
570            set_model_cooldown(
571                &agent_name,
572                &model_name,
573                config.silence_cooldown,
574                "silence_detected",
575            )
576            .await;
577            if let Some(result) = record_silence_detection(&agent_name, &model_name).await
578                && result.extended_cooldown_applied
579            {
580                extended_note = format!(
581                    " ({} silences in 24h -> extended cooldown {}s)",
582                    result.count, SILENCE_EXTENDED_COOLDOWN_SECS
583                );
584            }
585        }
586
587        // 3b. Short agent-level cooldown to force router to pick a different agent.
588        // Without this, the router picks the same agent with a different model,
589        // looping through all models (~2 min each) before the long agent cooldown kicks in.
590        if !agent_name.is_empty() {
591            set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
592        }
593
594        // 4. Pick a fallback agent and set to Routed (not New) to preserve progress.
595        // Setting to New would clear routing state and trigger a full LLM re-routing
596        // cycle, losing intermediate context. Routed skips re-routing and re-dispatches
597        // directly with the chosen fallback agent.
598        let task_eid = ExternalId(task_id.clone());
599
600        // Build available agents list and reroute chain for failover
601        let available: Vec<String> = engine::configured_agents()
602            .into_iter()
603            .filter(|a| cmd_cache::command_exists(a))
604            .collect();
605        let chain = response::get_reroute_chain(&task_id, &Some(Arc::clone(store)), repo).await;
606        let chain = response::update_reroute_chain(
607            &task_id,
608            &agent_name,
609            &chain,
610            &Some(Arc::clone(store)),
611            repo,
612        )
613        .await;
614
615        let (next_status, next_agent) = if let Some(fallback) =
616            response::pick_fallback_agent(&agent_name, &chain, &available)
617        {
618            tracing::info!(
619                task_id,
620                from = %agent_name,
621                to = %fallback,
622                "silence detection: failover to different agent, setting routed"
623            );
624            (Status::Routed, Some(fallback))
625        } else {
626            tracing::warn!(
627                task_id,
628                "silence detection: no fallback agents available, resetting to new for re-routing"
629            );
630            (Status::New, None)
631        };
632
633        if use_backend && let Some(ref st) = store_task {
634            // Fire-and-forget: label removals are cosmetic — tick should not block on them.
635            let stale_labels: Vec<String> = st
636                .labels
637                .iter()
638                .filter(|l| {
639                    l.starts_with("agent:")
640                        || l.starts_with("complexity:")
641                        || l.starts_with("model:")
642                })
643                .cloned()
644                .collect();
645            if !stale_labels.is_empty() {
646                let backend_clone = Arc::clone(backend);
647                let task_eid_clone = task_eid.clone();
648                tokio::spawn(async move {
649                    for label in &stale_labels {
650                        if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await {
651                            tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
652                        }
653                    }
654                });
655            }
656        }
657
658        if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
659            if let Some(ref fallback) = next_agent {
660                // Write the fallback agent into the store so dispatch can proceed
661                // directly without an LLM re-routing cycle. Clearing model forces
662                // model_for_complexity to pick the best available model for the new agent.
663                let _ = store_set_by_id(
664                    Some(store),
665                    store_id,
666                    &[
667                        ("agent", serde_json::json!(fallback)),
668                        ("model", serde_json::json!("")),
669                        (
670                            "last_error",
671                            serde_json::json!(format!(
672                                "silence detected after {}s, failing over to {}",
673                                config.silence_grace_period, fallback
674                            )),
675                        ),
676                    ],
677                )
678                .await;
679            } else {
680                let _ = store_set_by_id(
681                    Some(store),
682                    store_id,
683                    &[
684                        ("agent", serde_json::Value::Null),
685                        ("model", serde_json::Value::Null),
686                        (
687                            "last_error",
688                            serde_json::json!(format!(
689                                "silence detected after {}s, no fallback agents available",
690                                config.silence_grace_period
691                            )),
692                        ),
693                    ],
694                )
695                .await;
696            }
697        }
698
699        if let Err(e) = task_manager
700            .update_task_status(&task_eid, next_status)
701            .await
702        {
703            tracing::warn!(task_id, ?e, "failed to update silent task status");
704            continue;
705        }
706
707        // Log reroute activity
708        let details = serde_json::json!({
709            "failure_reason": "agent_silence",
710            "silence_duration_secs": config.silence_grace_period,
711            "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
712            "fallback_available": next_agent.is_some(),
713        });
714        store::store_log_activity(
715            &Some(Arc::clone(store)),
716            repo,
717            &task_id,
718            "rerouted",
719            Some("in_progress"),
720            Some(next_status.as_str()),
721            next_agent.as_deref(),
722            None,
723            Some(&details),
724        )
725        .await;
726
727        // 5. Post a comment explaining what happened
728        let action = if let Some(ref fallback) = next_agent {
729            format!("failing over to {fallback}")
730        } else {
731            "marking needs_review (no fallback agents)".to_string()
732        };
733        let comment = format!(
734            "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
735            chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
736            config.silence_grace_period,
737            agent_name,
738            model_name,
739            config.silence_cooldown,
740            extended_note,
741            agent_name,
742            SILENCE_AGENT_COOLDOWN_SECS,
743            action,
744            engine::orch_footer(),
745        );
746        if use_backend {
747            // Fire-and-forget: comment is informational — tick should not block on it.
748            let backend_clone = Arc::clone(backend);
749            let task_eid_clone = task_eid.clone();
750            tokio::spawn(async move {
751                if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
752                    tracing::warn!(
753                        task_id = task_eid_clone.0,
754                        ?e,
755                        "failed to post silence detection comment"
756                    );
757                }
758            });
759        }
760    }
761
762    Ok(())
763}
764
765fn should_use_backend(task_id: &str) -> bool {
766    !is_internal_id(task_id)
767}
768
769/// Phase 2 of tick: recover tasks stranded by missing or stale tmux state.
770/// - `in_progress` tasks → reset to `New` when their session is missing or the run timed out
771/// - `routed` tasks      → reset to `New` when an old live session keeps blocking re-dispatch
772/// - `in_review` tasks   → reset to `NeedsReview` when the review session disappeared
773#[expect(
774    clippy::too_many_arguments,
775    reason = "each argument is engine state the tick passes down"
776)]
777pub(crate) async fn tick_recover_stuck_tasks(
778    backend: &Arc<dyn ExternalBackend>,
779    tmux: &Arc<TmuxManager>,
780    repo: &str,
781    task_manager: &Arc<TaskManager>,
782    config: &EngineConfig,
783    store: &Arc<TaskStore>,
784    session_map: &HashMap<String, bool>,
785    dispatching: &Arc<DashMap<String, String>>,
786) -> anyhow::Result<()> {
787    let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
788    let tick_router_cell = OnceLock::new();
789
790    let in_progress = match task_manager
791        .list_external_by_status(Status::InProgress)
792        .await
793    {
794        Ok(tasks) => tasks,
795        Err(e) => {
796            tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
797            vec![]
798        }
799    };
800    for task in &in_progress {
801        let (since, task_config) = in_progress_stuck_clock(
802            store,
803            repo,
804            &task.id.0,
805            &task.updated_at,
806            session_alive(tmux, repo, &task.id.0, session_map),
807            config,
808            &tick_router_cell,
809        )
810        .await;
811        let Some(timing) = stuck_task_timing_from_map(
812            tmux,
813            repo,
814            &task.id.0,
815            &since,
816            &task_config,
817            "cannot parse updated_at, skipping stuck-task check",
818            session_map,
819        ) else {
820            continue;
821        };
822
823        // A dead tmux session with no `updated_at` refresh in a long time normally means
824        // the runner vanished (crashed, orch restarted). But it also matches the tail end
825        // of a *successful* run: the session exits the instant the agent finishes, while
826        // the runner is still polling for that exit (up to its own poll interval) and then
827        // writing results. If the runner's dispatch guard for this task is still held, the
828        // runner is alive and actively processing — let it finish rather than racing it.
829        if !timing.has_session {
830            let dispatch_key = format!("{repo}/{}", task.id.0);
831            if dispatching.contains_key(&dispatch_key) {
832                // Promoted from debug! to info!: the default log filter is "orch=info"
833                // (see main.rs), so this line — the only direct evidence that the guard
834                // check actually found and honored a held dispatch guard — was silently
835                // dropped in production. Every prior occurrence of the reclaim race
836                // (#3518/#3519/#3523/#3525/#3526) had to be diagnosed from timestamp
837                // correlation alone because this and the "dispatch guard
838                // claimed"/"released" lines never appeared in orch.log.
839                tracing::info!(
840                    task_id = task.id.0,
841                    dispatch_key,
842                    age_mins = timing.age.num_minutes(),
843                    "skipping stuck-task reclaim: runner still actively processing this task"
844                );
845                continue;
846            }
847        }
848
849        if timing.has_session {
850            tracing::warn!(
851                task_id = task.id.0,
852                age_mins = timing.age.num_minutes(),
853                threshold_mins = timing.threshold / 60,
854                "recovering stuck task: timed out with active session → new"
855            );
856            // The agent would keep working under a task that is routed again, and its session
857            // name would block the next dispatch
858            let session_name = tmux.session_name(repo, &task.id.0);
859            if let Err(e) = tmux.kill_session(&session_name).await {
860                tracing::warn!(
861                    task_id = task.id.0,
862                    session = %session_name,
863                    error = %e,
864                    "failed to kill stuck tmux session, will retry next tick"
865                );
866                continue;
867            }
868        } else {
869            tracing::warn!(
870                task_id = task.id.0,
871                dispatch_key = format!("{repo}/{}", task.id.0),
872                age_mins = timing.age.num_minutes(),
873                threshold_mins = timing.threshold / 60,
874                "recovering stuck task: no session found — reclaiming early → new"
875            );
876        }
877        // For stuck tasks with an active session, record agent/model failure + cooldown so the
878        // router picks a different agent/model on the next attempt. Without this, the router
879        // sees the agent as healthy and selects it again, causing the hang to repeat.
880        // (Mirrors the pattern in tick_detect_silent_agents lines 219–284.)
881        let mut cached_store_id: Option<i64> = None;
882        if timing.has_session {
883            let store_task = match store.resolve_task_id(repo, &task.id.0).await {
884                Ok(Some(store_id)) => {
885                    cached_store_id = Some(store_id);
886                    match store.get(store_id).await {
887                        Ok(t) => Some(t),
888                        Err(e) => {
889                            tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
890                            None
891                        }
892                    }
893                }
894                Ok(None) => None,
895                Err(e) => {
896                    tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
897                    None
898                }
899            };
900            let agent_name = store_task
901                .as_ref()
902                .and_then(|t| t.agent.clone())
903                .unwrap_or_default();
904            let model_name = store_task
905                .as_ref()
906                .and_then(|t| t.model.clone())
907                .unwrap_or_default();
908
909            if !agent_name.is_empty() && !model_name.is_empty() {
910                set_model_cooldown(
911                    &agent_name,
912                    &model_name,
913                    config.silence_cooldown,
914                    "silence_detected",
915                )
916                .await;
917                record_agent_failure_with_message(
918                    &agent_name,
919                    &format!(
920                        "stuck with active session after {}m",
921                        timing.age.num_minutes()
922                    ),
923                )
924                .await;
925            }
926            if !agent_name.is_empty() {
927                set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
928            }
929            tracing::warn!(
930                task_id = task.id.0,
931                agent = %agent_name,
932                model = %model_name,
933                cooldown_secs = config.silence_cooldown,
934                "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
935            );
936        }
937
938        // Remove stale agent/model labels so the LLM router re-routes properly.
939        // Fire-and-forget: cosmetic label operation, tick should not block on it.
940        {
941            let stale_labels: Vec<String> = task
942                .labels
943                .iter()
944                .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
945                .cloned()
946                .collect();
947            if !stale_labels.is_empty() {
948                let backend_clone = Arc::clone(backend);
949                let task_id_clone = task.id.clone();
950                tokio::spawn(async move {
951                    for label in &stale_labels {
952                        if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
953                            tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
954                        }
955                    }
956                });
957            }
958        }
959        let resolved_store_id = match cached_store_id {
960            Some(id) => Some(id),
961            None => match store.resolve_task_id(repo, &task.id.0).await {
962                Ok(Some(id)) => Some(id),
963                Ok(None) => {
964                    tracing::warn!(
965                        task_id = %task.id.0,
966                        repo,
967                        "resolve_task_id returned None during stuck-task recovery"
968                    );
969                    None
970                }
971                Err(e) => {
972                    tracing::error!(
973                        task_id = %task.id.0,
974                        repo,
975                        error = %e,
976                        "resolve_task_id failed during stuck-task recovery"
977                    );
978                    None
979                }
980            },
981        };
982        if let Some(store_id) = resolved_store_id {
983            if let Err(e) = store_set_by_id(
984                Some(store),
985                store_id,
986                &[
987                    ("agent", serde_json::Value::Null),
988                    ("model", serde_json::Value::Null),
989                    ("route_attempts", serde_json::json!(0)),
990                ],
991            )
992            .await
993            {
994                tracing::warn!(
995                    task_id = task.id.0,
996                    ?e,
997                    "failed to clear routing fields for stuck task — will retry next tick"
998                );
999                continue;
1000            }
1001            finalize_open_runs_for_recovery(
1002                store,
1003                store_id,
1004                if timing.has_session {
1005                    "stuck-task recovery: active session killed"
1006                } else {
1007                    "stuck-task recovery: no session found"
1008                },
1009            )
1010            .await;
1011        }
1012        if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
1013            tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
1014            continue;
1015        }
1016
1017        // Log reroute activity for stuck task recovery
1018        let details = serde_json::json!({
1019            "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1020            "age_minutes": timing.age.num_minutes(),
1021            "had_session": timing.has_session,
1022        });
1023        store::store_log_activity(
1024            &Some(Arc::clone(store)),
1025            repo,
1026            &task.id.0,
1027            "rerouted",
1028            Some("in_progress"),
1029            Some("new"),
1030            None::<&str>,
1031            None::<&str>,
1032            Some(&details),
1033        )
1034        .await;
1035
1036        let reason = if timing.has_session {
1037            format!(
1038                "timed out after {}m with active session (cleared agent for re-routing)",
1039                timing.age.num_minutes()
1040            )
1041        } else {
1042            format!(
1043                "no session found — reclaiming early after {}m (cleared agent for re-routing)",
1044                timing.age.num_minutes()
1045            )
1046        };
1047        // Fire-and-forget: comment is informational — tick should not block on it.
1048        {
1049            let backend_clone = Arc::clone(backend);
1050            let task_id_clone = task.id.clone();
1051            let body = format!(
1052                "[{}] recovered: stuck in_progress — {}{}",
1053                chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1054                reason,
1055                engine::orch_footer()
1056            );
1057            tokio::spawn(async move {
1058                if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
1059                    tracing::warn!(
1060                        task_id = task_id_clone.0,
1061                        ?e,
1062                        "failed to post stuck-task recovery comment"
1063                    );
1064                }
1065            });
1066        }
1067    }
1068
1069    let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
1070        Ok(tasks) => tasks,
1071        Err(e) => {
1072            tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
1073            vec![]
1074        }
1075    };
1076    match task_manager
1077        .list_internal_by_status(TaskStatus::Routed)
1078        .await
1079    {
1080        Ok(tasks) => routed.extend(tasks),
1081        Err(e) => tracing::warn!(
1082            ?e,
1083            "failed to list internal routed tasks for stuck recovery"
1084        ),
1085    }
1086    for task in &routed {
1087        let Some(timing) = stuck_task_timing_from_map(
1088            tmux,
1089            repo,
1090            &task.id.0,
1091            &task.updated_at,
1092            config,
1093            "cannot parse updated_at, skipping routed stuck-task check",
1094            session_map,
1095        ) else {
1096            continue;
1097        };
1098
1099        if !timing.has_session {
1100            continue;
1101        }
1102
1103        let session_name = tmux.session_name(repo, &task.id.0);
1104        if let Err(e) = tmux.kill_session(&session_name).await {
1105            tracing::warn!(
1106                task_id = task.id.0,
1107                session = %session_name,
1108                error = %e,
1109                "failed to kill stale tmux session blocking routed task"
1110            );
1111            continue;
1112        }
1113
1114        recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
1115    }
1116
1117    // Recover internal (SQLite) tasks stuck in in_progress.
1118    // These have no GitHub labels or comments — just reset the DB status to New.
1119    let internal_in_progress = match task_manager
1120        .list_internal_by_status(DbStatus::InProgress)
1121        .await
1122    {
1123        Ok(tasks) => tasks,
1124        Err(e) => {
1125            tracing::warn!(
1126                ?e,
1127                "failed to list internal in_progress tasks for stuck recovery"
1128            );
1129            vec![]
1130        }
1131    };
1132    for task in &internal_in_progress {
1133        let task_id = task.id.0.clone();
1134        let (since, task_config) = in_progress_stuck_clock(
1135            store,
1136            repo,
1137            &task_id,
1138            &task.updated_at,
1139            session_alive(tmux, repo, &task_id, session_map),
1140            config,
1141            &tick_router_cell,
1142        )
1143        .await;
1144        let Some(timing) = stuck_task_timing_from_map(
1145            tmux,
1146            repo,
1147            &task_id,
1148            &since,
1149            &task_config,
1150            "cannot parse updated_at, skipping stuck internal-task check",
1151            session_map,
1152        ) else {
1153            continue;
1154        };
1155
1156        // See the comment on the external in-progress loop above: a dead session with a
1157        // stale `updated_at` also matches the tail end of a run that just finished
1158        // successfully. Skip the reclaim while the runner's dispatch guard for this task
1159        // is still held — this is the exact race that discards completed daily/scheduled
1160        // internal tasks whose total runtime exceeds `no_session_stuck_timeout`.
1161        if !timing.has_session {
1162            let dispatch_key = format!("{repo}/{task_id}");
1163            if dispatching.contains_key(&dispatch_key) {
1164                // Promoted from debug! to info! (issue #3526): the default log filter is
1165                // "orch=info" (main.rs), so this line — along with the DispatchGuard
1166                // claim/release lines in dispatch_guard.rs and dispatch_tasks_for_repo —
1167                // never reached orch.log in production. Every occurrence of the reclaim
1168                // race (#3518, #3519, #3523, #3525, #3526) had to be reasoned about from
1169                // timestamp correlation across unrelated log lines instead of directly
1170                // observing whether the guard was held, because the one log line that
1171                // would prove it was silently filtered out. Promoting these closes that
1172                // observability gap so the next occurrence carries conclusive evidence.
1173                tracing::info!(
1174                    task_id,
1175                    dispatch_key,
1176                    age_mins = timing.age.num_minutes(),
1177                    "skipping stuck-task reclaim: runner still actively processing this task"
1178                );
1179                continue;
1180            }
1181        }
1182
1183        if timing.has_session {
1184            tracing::warn!(
1185                task_id,
1186                age_mins = timing.age.num_minutes(),
1187                threshold_mins = timing.threshold / 60,
1188                "recovering stuck task: timed out with active session → new"
1189            );
1190            let session_name = tmux.session_name(repo, &task_id);
1191            if let Err(e) = tmux.kill_session(&session_name).await {
1192                tracing::warn!(
1193                    task_id,
1194                    session = %session_name,
1195                    error = %e,
1196                    "failed to kill stuck tmux session, will retry next tick"
1197                );
1198                continue;
1199            }
1200        } else {
1201            tracing::warn!(
1202                task_id,
1203                dispatch_key = format!("{repo}/{task_id}"),
1204                age_mins = timing.age.num_minutes(),
1205                threshold_mins = timing.threshold / 60,
1206                "recovering stuck task: no session found — reclaiming early → new"
1207            );
1208        }
1209        // Apply agent/model cooldown for internal stuck tasks with active sessions,
1210        // mirroring the external path (lines 501-546). Without this, the router picks
1211        // the same agent/model that caused the hang, creating an infinite loop.
1212        let mut cached_store_id: Option<i64> = None;
1213        if timing.has_session {
1214            let store_task = match store.resolve_task_id(repo, &task_id).await {
1215                Ok(Some(store_id)) => {
1216                    cached_store_id = Some(store_id);
1217                    match store.get(store_id).await {
1218                        Ok(t) => Some(t),
1219                        Err(e) => {
1220                            tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1221                            None
1222                        }
1223                    }
1224                }
1225                Ok(None) => None,
1226                Err(e) => {
1227                    tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1228                    None
1229                }
1230            };
1231            let agent_name = store_task
1232                .as_ref()
1233                .and_then(|t| t.agent.clone())
1234                .unwrap_or_default();
1235            let model_name = store_task
1236                .as_ref()
1237                .and_then(|t| t.model.clone())
1238                .unwrap_or_default();
1239
1240            if !agent_name.is_empty() && !model_name.is_empty() {
1241                set_model_cooldown(
1242                    &agent_name,
1243                    &model_name,
1244                    config.silence_cooldown,
1245                    "silence_detected",
1246                )
1247                .await;
1248                record_agent_failure_with_message(
1249                    &agent_name,
1250                    &format!(
1251                        "internal task stuck with active session after {}m",
1252                        timing.age.num_minutes()
1253                    ),
1254                )
1255                .await;
1256            }
1257            if !agent_name.is_empty() {
1258                set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1259            }
1260            tracing::warn!(
1261                task_id,
1262                agent = %agent_name,
1263                model = %model_name,
1264                cooldown_secs = config.silence_cooldown,
1265                "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1266            );
1267        }
1268        // Reset routing state so the LLM router is used on the next attempt
1269        // (same reset that external tasks perform).
1270        let resolved_store_id = match cached_store_id {
1271            Some(id) => Some(id),
1272            None => match store.resolve_task_id(repo, &task_id).await {
1273                Ok(Some(id)) => Some(id),
1274                Ok(None) => {
1275                    tracing::warn!(
1276                        task_id = %task_id,
1277                        repo,
1278                        "resolve_task_id returned None during stuck internal-task recovery"
1279                    );
1280                    None
1281                }
1282                Err(e) => {
1283                    tracing::error!(
1284                        task_id = %task_id,
1285                        repo,
1286                        error = %e,
1287                        "resolve_task_id failed during stuck internal-task recovery"
1288                    );
1289                    None
1290                }
1291            },
1292        };
1293        if let Some(store_id) = resolved_store_id {
1294            if let Err(e) = store_set_by_id(
1295                Some(store),
1296                store_id,
1297                &[
1298                    ("agent", serde_json::Value::Null),
1299                    ("model", serde_json::Value::Null),
1300                    ("route_attempts", serde_json::json!(0)),
1301                ],
1302            )
1303            .await
1304            {
1305                tracing::warn!(
1306                    task_id,
1307                    ?e,
1308                    "failed to clear routing fields for stuck internal task — will retry next tick"
1309                );
1310                continue;
1311            }
1312            finalize_open_runs_for_recovery(
1313                store,
1314                store_id,
1315                if timing.has_session {
1316                    "stuck-task recovery: internal active session killed"
1317                } else {
1318                    "stuck-task recovery: internal no session found"
1319                },
1320            )
1321            .await;
1322        }
1323        if let Err(e) = task_manager
1324            .update_task_status(&ExternalId(task_id.clone()), Status::New)
1325            .await
1326        {
1327            tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1328        }
1329    }
1330
1331    // Recover tasks stuck in in_review, external and internal in one sweep. Reset to NeedsReview
1332    // (not New) so the review agent re-triggers without clearing the routing state.
1333    //
1334    // This is the only periodic owner of the case where the review session is gone. The sync
1335    // sweep used to reset the same tasks after one minute, which beat a finished review's result
1336    // to the engine and dropped it as stale, and left this longer limit unreachable
1337    let in_review = match store.list_by_status(repo, TaskStatus::InReview).await {
1338        Ok(v) => v,
1339        Err(e) => {
1340            tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1341            vec![]
1342        }
1343    };
1344    // A review agent ends its tmux session before its result reaches the engine, so the
1345    // no-session limit for a review has to be longer than for a task agent. The floor in
1346    // `EngineConfig::from_config` keeps it above the review agent's own timeout
1347    let in_review_config = engine::EngineConfig {
1348        no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1349        ..config.clone()
1350    };
1351    for task in &in_review {
1352        let Some(task_id) = task.external_id.clone() else {
1353            tracing::warn!(
1354                store_id = task.id,
1355                "in_review task has no external id, skipping stuck recovery"
1356            );
1357            continue;
1358        };
1359
1360        // The dispatch flow owns a task while it holds the guard
1361        if dispatching.contains_key(&format!("{repo}/{task_id}")) {
1362            tracing::debug!(
1363                task_id,
1364                "task locked by dispatch flow, skipping stuck check"
1365            );
1366            continue;
1367        }
1368
1369        // A recorded decision means the review finished and the task waits on the merge, not on
1370        // a review agent
1371        let (entered_at, decided) = in_review_stuck_entry(store, task.id, &task.updated_at).await;
1372        if decided {
1373            tracing::debug!(
1374                task_id,
1375                "in_review task already has a review decision, waiting on merge"
1376            );
1377            continue;
1378        }
1379
1380        let review_task_id = format!("{task_id}-review");
1381        let Some(timing) = stuck_task_timing_from_map(
1382            tmux,
1383            repo,
1384            &review_task_id,
1385            &entered_at,
1386            &in_review_config,
1387            "cannot parse in_review timestamp, skipping stuck in_review check",
1388            session_map,
1389        ) else {
1390            continue;
1391        };
1392
1393        tracing::warn!(
1394            task_id,
1395            age_mins = timing.age.num_minutes(),
1396            threshold_mins = timing.threshold / 60,
1397            has_session = timing.has_session,
1398            "recovering stuck in_review task — resetting to needs_review"
1399        );
1400
1401        match task_manager
1402            .update_task_status_if(
1403                &ExternalId(task_id.clone()),
1404                Status::NeedsReview,
1405                Status::InReview,
1406            )
1407            .await
1408        {
1409            Err(e) => {
1410                tracing::warn!(task_id, ?e, "failed to reset stuck in_review task status");
1411                continue;
1412            }
1413            Ok(false) => {
1414                tracing::debug!(
1415                    task_id,
1416                    "stuck in_review reset skipped — task already transitioned"
1417                );
1418                continue;
1419            }
1420            Ok(true) => {}
1421        }
1422
1423        // A dead or hung session is an infrastructure event, a tmux crash or a restart, not an
1424        // agent parse failure. Keeping the counter would spend the next cycle's quota
1425        if let Err(e) = store
1426            .set_fields(task.id, &[("review_agent_failures", serde_json::json!(0))])
1427            .await
1428        {
1429            tracing::warn!(task_id, ?e, "failed to reset review_agent_failures");
1430        }
1431
1432        finalize_open_runs_for_recovery(
1433            store,
1434            task.id,
1435            if timing.has_session {
1436                "stuck-task recovery: in_review session killed"
1437            } else {
1438                "stuck-task recovery: in_review no session found"
1439            },
1440        )
1441        .await;
1442
1443        let details = serde_json::json!({
1444            "failure_reason": "stuck_in_review",
1445            "age_minutes": timing.age.num_minutes(),
1446            "had_session": timing.has_session,
1447        });
1448        store::store_log_activity(
1449            &Some(Arc::clone(store)),
1450            repo,
1451            &task_id,
1452            "rerouted",
1453            Some("in_review"),
1454            Some("needs_review"),
1455            None::<&str>,
1456            None::<&str>,
1457            Some(&details),
1458        )
1459        .await;
1460
1461        // An internal task has no issue to comment on
1462        if task.origin == "internal" {
1463            continue;
1464        }
1465
1466        if let Err(e) = backend
1467            .post_comment(
1468                &ExternalId(task_id.clone()),
1469                &format!(
1470                    "[{}] recovered: stuck in_review — {} after {}m, resetting to needs_review{}",
1471                    chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1472                    if timing.has_session {
1473                        "review session hung"
1474                    } else {
1475                        "no session found"
1476                    },
1477                    timing.age.num_minutes(),
1478                    engine::orch_footer()
1479                ),
1480            )
1481            .await
1482        {
1483            tracing::warn!(
1484                task_id,
1485                ?e,
1486                "failed to post stuck in_review recovery comment"
1487            );
1488        }
1489    }
1490
1491    Ok(())
1492}
1493
1494/// External tasks stuck in_progress in a repo no project ticks, such as a project removed from
1495/// the config. Store-wide, so `tick_all` runs it once per tick rather than once per project
1496pub(crate) async fn recover_untracked_repo_stuck_tasks(
1497    tmux: &Arc<TmuxManager>,
1498    config: &EngineConfig,
1499    store: &Arc<TaskStore>,
1500    session_map: &HashMap<String, bool>,
1501    dispatching: &Arc<DashMap<String, String>>,
1502    active_repos: &HashSet<String>,
1503) {
1504    let all_external_in_progress = match store
1505        .list_all_by_status_global(TaskStatus::InProgress)
1506        .await
1507    {
1508        Ok(tasks) => tasks,
1509        Err(e) => {
1510            tracing::warn!(
1511                ?e,
1512                "failed to list global in_progress tasks for cross-repo stuck recovery"
1513            );
1514            vec![]
1515        }
1516    };
1517    for task in all_external_in_progress
1518        .iter()
1519        .filter(|t| t.origin != "internal" && !active_repos.contains(&t.repo))
1520    {
1521        let external_id = task
1522            .external_id
1523            .clone()
1524            .unwrap_or_else(|| task.id.to_string());
1525        let Some(timing) = stuck_task_timing_from_map(
1526            tmux,
1527            &task.repo,
1528            &external_id,
1529            &task.updated_at,
1530            config,
1531            "cannot parse updated_at, skipping cross-repo stuck-task check",
1532            session_map,
1533        ) else {
1534            continue;
1535        };
1536
1537        // See the comment on the in-progress loop in `tick_recover_stuck_tasks`: a dead session with a stale
1538        // `updated_at` also matches the tail end of a run that just finished successfully.
1539        // Skip the reclaim while the runner's dispatch guard for this task is still held.
1540        if !timing.has_session {
1541            let dispatch_key = format!("{}/{external_id}", task.repo);
1542            if dispatching.contains_key(&dispatch_key) {
1543                // See the same promotion note on the internal-task loop in `tick_recover_stuck_tasks`: this was
1544                // debug! (silently dropped under the default "orch=info" filter), leaving
1545                // every prior reclaim-race occurrence undiagnosable from logs alone.
1546                tracing::info!(
1547                    task_id = task.id,
1548                    repo = %task.repo,
1549                    dispatch_key,
1550                    age_mins = timing.age.num_minutes(),
1551                    "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1552                );
1553                continue;
1554            }
1555        }
1556
1557        if timing.has_session {
1558            tracing::warn!(
1559                task_id = task.id,
1560                repo = %task.repo,
1561                age_mins = timing.age.num_minutes(),
1562                threshold_mins = timing.threshold / 60,
1563                "recovering cross-repo stuck task: timed out with active session → new"
1564            );
1565        } else {
1566            tracing::warn!(
1567                task_id = task.id,
1568                repo = %task.repo,
1569                dispatch_key = format!("{}/{external_id}", task.repo),
1570                age_mins = timing.age.num_minutes(),
1571                threshold_mins = timing.threshold / 60,
1572                "recovering cross-repo stuck task: no session found — reclaiming early → new"
1573            );
1574        }
1575
1576        if let Err(e) = store_set_by_id(
1577            Some(store),
1578            task.id,
1579            &[
1580                ("agent", serde_json::Value::Null),
1581                ("model", serde_json::Value::Null),
1582                ("route_attempts", serde_json::json!(0)),
1583            ],
1584        )
1585        .await
1586        {
1587            tracing::warn!(
1588                task_id = task.id,
1589                repo = %task.repo,
1590                ?e,
1591                "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1592            );
1593            continue;
1594        }
1595
1596        finalize_open_runs_for_recovery(
1597            store,
1598            task.id,
1599            if timing.has_session {
1600                "stuck-task recovery: cross-repo active session killed"
1601            } else {
1602                "stuck-task recovery: cross-repo no session found"
1603            },
1604        )
1605        .await;
1606
1607        if let Err(e) = store.update_status(task.id, TaskStatus::New).await {
1608            tracing::warn!(
1609                task_id = task.id,
1610                repo = %task.repo,
1611                ?e,
1612                "failed to reset cross-repo stuck task status"
1613            );
1614            continue;
1615        }
1616
1617        let details = serde_json::json!({
1618            "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1619            "age_minutes": timing.age.num_minutes(),
1620            "had_session": timing.has_session,
1621            "scope": "cross_repo_global_sweep",
1622        });
1623        store::store_log_activity(
1624            &Some(Arc::clone(store)),
1625            &task.repo,
1626            &external_id,
1627            "rerouted",
1628            Some("in_progress"),
1629            Some("new"),
1630            None::<&str>,
1631            None::<&str>,
1632            Some(&details),
1633        )
1634        .await;
1635    }
1636}
1637
1638/**
1639 * The three store-wide recovery sweeps, run once per tick rather than once per project.
1640 *
1641 * Each one reads every repo, so a per-project call duplicates the work. The NeedsReview
1642 * sweep also reads `needs_review_refires` and then increments it, so concurrent per-project
1643 * passes all see the same count and four projects burn four of the five refire attempts in
1644 * one tick (#3864)
1645 **/
1646pub(crate) async fn tick_global_sweeps(
1647    task_manager: &Arc<TaskManager>,
1648    store: &Arc<TaskStore>,
1649    router: &Router,
1650    dispatching: &Arc<DashMap<String, String>>,
1651    auto_merge_in_flight: &Arc<DashSet<String>>,
1652    active_repos: &HashSet<String>,
1653) {
1654    // Global sweep for CI-failure blocked tasks from inactive or removed projects.
1655    // auto_unblock_blocked_tasks runs inside sync_tick and is scoped to the active repo.
1656    // Tasks blocked for projects no longer in orch config are never processed there.
1657    // This sweep covers all repos so stale CI-failure blocks are eventually resolved.
1658    if let Err(e) = sync::auto_unblock_ci_failure_blocked_tasks_global(
1659        task_manager,
1660        store,
1661        auto_merge_in_flight,
1662    )
1663    .await
1664    {
1665        tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1666    }
1667
1668    // Re-fire stale NeedsReview tasks and escalate persistently-stuck ones to
1669    // Blocked, across all repos
1670    if let Err(e) = sync::refire_and_escalate_stale_needs_review_global(
1671        task_manager,
1672        store,
1673        router,
1674        dispatching,
1675        active_repos,
1676    )
1677    .await
1678    {
1679        tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1680    }
1681
1682    // Recover rebroadcast-blocked tasks once a review agent is routable, across
1683    // all repos
1684    if let Err(e) = sync::auto_recover_rebroadcast_blocked_tasks_global(
1685        router,
1686        task_manager,
1687        store,
1688        active_repos,
1689    )
1690    .await
1691    {
1692        tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1693    }
1694}
1695
1696/// Phase 3a of tick: route status:new tasks to an agent and transition them to status:routed.
1697pub(crate) async fn tick_route_tasks(
1698    backend: &Arc<dyn ExternalBackend>,
1699    task_manager: &Arc<TaskManager>,
1700    router: &mut Router,
1701    store: &Arc<TaskStore>,
1702    repo: &str,
1703    routing_quota: &mut RoutingTickQuota,
1704) -> anyhow::Result<()> {
1705    let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1706    if let Err(err) = dependencies::apply_claim_gates(store).await {
1707        tracing::warn!(err = %err, "dependency claim gate failed");
1708    }
1709    if routing_quota.is_exhausted() {
1710        tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1711        return Ok(());
1712    }
1713
1714    // Global GitHub 5xx circuit breaker — skip routing during sustained GitHub outages
1715    // to avoid routing-heavy retry storms. Tasks will remain in 'new' status and be
1716    // retried when the circuit closes.
1717    if is_github_circuit_open() {
1718        let remaining = github_circuit_remaining_secs();
1719        tracing::info!(
1720            remaining_secs = remaining,
1721            "GitHub 5xx circuit breaker open — skipping routing phase"
1722        );
1723        return Ok(());
1724    }
1725    let new_tasks = task_manager.list_routable().await?;
1726    let routable: Vec<&ExternalTask> = new_tasks
1727        .iter()
1728        .filter(|t| !skips_agent(&t.labels))
1729        .collect();
1730
1731    // Pre-emptive health check: refresh degraded-agent flags once per tick,
1732    // not per-task (previously this ran inside route() causing N redundant
1733    // DB queries when routing N tasks in the same tick).
1734    if !routable.is_empty() {
1735        let start = Instant::now();
1736        router.refresh_health(store).await;
1737        tracing::debug!(
1738            duration_ms = start.elapsed().as_millis(),
1739            "router.refresh_health completed"
1740        );
1741    }
1742
1743    // Limit routing to the remaining global quota for this outer engine tick.
1744    for task in routable.into_iter().take(routing_quota.remaining()) {
1745        if !routing_quota.consume_one() {
1746            break;
1747        }
1748        let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1749
1750        let task_start = Instant::now();
1751        match router.route(task, store, repo).await {
1752            Ok(result) => {
1753                tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1754                // Store route result in store
1755                if let Err(e) = router
1756                    .store_route_result(&task.id.0, &result, store, repo)
1757                    .await
1758                {
1759                    tracing::warn!(
1760                        task_id = task.id.0,
1761                        ?e,
1762                        "failed to store route result; skipping Routed transition"
1763                    );
1764                    continue;
1765                }
1766
1767                if is_internal_id(&task.id.0) {
1768                    // Internal tasks: update DB status, skip GitHub label ops.
1769                    if let Err(e) = task_manager
1770                        .update_task_status(&task.id, Status::Routed)
1771                        .await
1772                    {
1773                        tracing::warn!(
1774                            task_id = task.id.0,
1775                            ?e,
1776                            "failed to set internal status:routed"
1777                        );
1778                        continue;
1779                    }
1780                } else {
1781                    // Fire-and-forget: label operations are cosmetic (routing already
1782                    // persisted in DB). Spawning them prevents blocking the tick loop
1783                    // on 6-9 sequential HTTP calls (~3-9s overhead per routed task).
1784                    let backend_clone = Arc::clone(backend);
1785                    let task_id_clone = task.id.clone();
1786                    let task_labels: Vec<String> = task.labels.clone();
1787                    let new_labels = {
1788                        let mut labels = vec![
1789                            format!("agent:{}", result.agent),
1790                            format!("complexity:{}", result.complexity),
1791                        ];
1792                        if let Some(ref model) = result.model {
1793                            labels.push(format!("model:{model}"));
1794                        }
1795                        labels
1796                    };
1797                    tokio::spawn(async move {
1798                        // Remove old agent/complexity/model labels
1799                        for label in &task_labels {
1800                            if (label.starts_with("agent:")
1801                                || label.starts_with("complexity:")
1802                                || label.starts_with("model:"))
1803                                && let Err(e) =
1804                                    backend_clone.remove_label(&task_id_clone, label).await
1805                            {
1806                                tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1807                            }
1808                        }
1809                        // Add new labels
1810                        if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1811                        {
1812                            tracing::warn!(
1813                                task_id = task_id_clone.0,
1814                                ?e,
1815                                "failed to set routing labels"
1816                            );
1817                        }
1818                    });
1819
1820                    // Transition to routed
1821                    if let Err(e) = task_manager
1822                        .update_task_status(&task.id, Status::Routed)
1823                        .await
1824                    {
1825                        tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1826                        continue;
1827                    }
1828                }
1829
1830                // Dual-write: upsert task + store route result in SQLite
1831                match store.ensure_external_task(repo, task).await {
1832                    Ok(store_id) => {
1833                        let profile_json =
1834                            serde_json::to_string(&result.profile).unwrap_or_default();
1835                        let skills_json =
1836                            serde_json::to_string(&result.selected_skills).unwrap_or_default();
1837                        if let Err(e) = store
1838                            .store_route(&crate::store::StoreRoute {
1839                                id: store_id,
1840                                agent: &result.agent,
1841                                model: result.model.as_deref(),
1842                                complexity: &result.complexity,
1843                                estimate: result.estimate,
1844                                reason: &result.reason,
1845                                profile: &profile_json,
1846                                skills: &skills_json,
1847                            })
1848                            .await
1849                        {
1850                            tracing::debug!(
1851                                task_id = task.id.0,
1852                                ?e,
1853                                "dual-write: store_route failed"
1854                            );
1855                        }
1856                        if let Err(e) = store.update_status(store_id, TaskStatus::Routed).await {
1857                            tracing::debug!(
1858                                task_id = task.id.0,
1859                                ?e,
1860                                "dual-write: update_status failed"
1861                            );
1862                        }
1863                        // Sync the estimate to GitHub Projects if the router assigned one.
1864                        // Fire-and-forget: do NOT await inline — GraphQL mutations can take
1865                        // up to 30 s and would block the entire tick loop on every routed task.
1866                        if result.estimate > 0 {
1867                            let backend_clone = Arc::clone(backend);
1868                            let task_id = task.id.clone();
1869                            let estimate = result.estimate;
1870                            tokio::spawn(async move {
1871                                if let Err(e) = backend_clone
1872                                    .sync_estimate_to_project(&task_id, estimate)
1873                                    .await
1874                                {
1875                                    tracing::debug!(
1876                                        task_id = task_id.0,
1877                                        estimate,
1878                                        err = %e,
1879                                        "dual-write: estimate project sync failed"
1880                                    );
1881                                }
1882                            });
1883                        }
1884                    }
1885                    Err(e) => {
1886                        tracing::debug!(
1887                            task_id = task.id.0,
1888                            ?e,
1889                            "dual-write: ensure_external_task failed"
1890                        );
1891                    }
1892                }
1893
1894                if let Some(ref warning) = result.warning {
1895                    tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1896                }
1897
1898                tracing::info!(
1899                    task_id = task.id.0,
1900                    agent = %result.agent,
1901                    complexity = %result.complexity,
1902                    reason = %result.reason,
1903                    "task routed"
1904                );
1905            }
1906            Err(e) => {
1907                if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1908                    tracing::error!(
1909                        task_id = %task.id.0,
1910                        scope = %cooled.scope(),
1911                        remaining_secs = ?cooled.remaining_secs(),
1912                        degraded = ?cooled.degraded(),
1913                        "ALL AGENTS COOLED — no agent available to route this task. \
1914                         Task stays in 'new' and will be retried on every tick. \
1915                         Check `orch cooldown list` to see what is blocking dispatch."
1916                    );
1917                    continue;
1918                }
1919                if let Some(waiting) = e.downcast_ref::<WaitingOnDependency>() {
1920                    tracing::info!(
1921                        task_id = %task.id.0,
1922                        blockers = %waiting.blockers.join(", "),
1923                        "task is waiting on a dependency"
1924                    );
1925                    continue;
1926                }
1927                tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1928            }
1929        }
1930    }
1931    Ok(())
1932}
1933
1934/// Global routing sweep: route status:new tasks from inactive or removed repos.
1935///
1936/// `tick_route_tasks` is scoped to the active repo per-tick and never processes
1937/// tasks whose `repo` field differs from the current project. When a project is
1938/// removed from the orch config, its tasks accumulate in `new` indefinitely
1939/// because no tick ever runs for that repo. This sweep covers all repos so
1940/// orphaned new tasks are eventually routed.
1941///
1942/// `active_repos` must be the full set of currently-configured project repos,
1943/// not just the repo of the tick that called this — mirroring
1944/// `dispatch_routed_tasks_global` (see its doc comment for the "racing the real
1945/// per-repo dispatch" failure mode this guards against). Filtering on
1946/// `t.repo != current_repo` alone (the previous behavior) made every *other*
1947/// active repo's new tasks look "orphaned" too and routed them a tick early,
1948/// through the wrong repo's tick.
1949pub(crate) async fn route_new_tasks_global(
1950    router: &mut Router,
1951    store: &Arc<TaskStore>,
1952    active_repos: &HashSet<String>,
1953    routing_quota: &mut RoutingTickQuota,
1954) -> anyhow::Result<()> {
1955    if routing_quota.is_exhausted() {
1956        tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1957        return Ok(());
1958    }
1959
1960    let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1961    let orphaned: Vec<&Task> = all_new
1962        .iter()
1963        .filter(|t| !active_repos.contains(&t.repo))
1964        .filter(|t| !skips_agent(&t.labels))
1965        .collect();
1966
1967    if orphaned.is_empty() {
1968        return Ok(());
1969    }
1970
1971    tracing::info!(
1972        count = orphaned.len(),
1973        "global routing sweep: found new tasks from inactive repos"
1974    );
1975
1976    router.refresh_health(store).await;
1977
1978    for task in orphaned.into_iter().take(routing_quota.remaining()) {
1979        if !routing_quota.consume_one() {
1980            break;
1981        }
1982        let ext_task = tasks::store_task_to_external(task);
1983        let task_start = Instant::now();
1984        match router.route(&ext_task, store, &task.repo).await {
1985            Ok(result) => {
1986                tracing::debug!(
1987                    task_id = %ext_task.id.0,
1988                    repo = %task.repo,
1989                    duration_ms = task_start.elapsed().as_millis(),
1990                    "global sweep: route completed"
1991                );
1992                let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1993                let skills_json =
1994                    serde_json::to_string(&result.selected_skills).unwrap_or_default();
1995                if let Err(e) = store
1996                    .store_route(&StoreRoute {
1997                        id: task.id,
1998                        agent: &result.agent,
1999                        model: result.model.as_deref(),
2000                        complexity: &result.complexity,
2001                        estimate: result.estimate,
2002                        reason: &result.reason,
2003                        profile: &profile_json,
2004                        skills: &skills_json,
2005                    })
2006                    .await
2007                {
2008                    tracing::warn!(
2009                        task_id = %ext_task.id.0,
2010                        repo = %task.repo,
2011                        err = %e,
2012                        "global sweep: failed to store route result; skipping status update"
2013                    );
2014                    continue;
2015                }
2016                match store.update_status(task.id, TaskStatus::Routed).await {
2017                    Err(e) => {
2018                        tracing::warn!(
2019                            task_id = %ext_task.id.0,
2020                            repo = %task.repo,
2021                            err = %e,
2022                            "global sweep: failed to set status:routed"
2023                        );
2024                    }
2025                    _ => {
2026                        tracing::info!(
2027                            task_id = %ext_task.id.0,
2028                            repo = %task.repo,
2029                            agent = %result.agent,
2030                            complexity = %result.complexity,
2031                            "global sweep: task routed"
2032                        );
2033                    }
2034                }
2035                if let Some(ref warning) = result.warning {
2036                    tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
2037                }
2038            }
2039            Err(e) => {
2040                if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
2041                    tracing::error!(
2042                        task_id = %ext_task.id.0,
2043                        repo = %task.repo,
2044                        scope = %cooled.scope(),
2045                        remaining_secs = ?cooled.remaining_secs(),
2046                        degraded = ?cooled.degraded(),
2047                        "global sweep: ALL AGENTS COOLED — task stays in 'new'"
2048                    );
2049                    continue;
2050                }
2051                if let Some(waiting) = e.downcast_ref::<WaitingOnDependency>() {
2052                    tracing::info!(
2053                        task_id = %ext_task.id.0,
2054                        repo = %task.repo,
2055                        blockers = %waiting.blockers.join(", "),
2056                        "global sweep: task is waiting on a dependency"
2057                    );
2058                    continue;
2059                }
2060                tracing::error!(
2061                    task_id = %ext_task.id.0,
2062                    repo = %task.repo,
2063                    err = ?e,
2064                    "global sweep: routing failed, skipping task"
2065                );
2066            }
2067        }
2068    }
2069
2070    Ok(())
2071}
2072
2073/// Phase 3b of tick: spawn agents for all status:routed tasks up to the parallel limit.
2074#[derive(Debug, Clone, Copy)]
2075pub(crate) struct DispatchMode {
2076    pub(crate) is_degraded: bool,
2077    pub(crate) healthy_agents: usize,
2078    pub(crate) threshold: usize,
2079}
2080
2081pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
2082    let threshold = router_config::min_healthy_agents_threshold();
2083    let healthy_agents = router.healthy_agent_count("simple");
2084    let is_degraded = router.is_degraded(threshold);
2085    DispatchMode {
2086        is_degraded,
2087        healthy_agents,
2088        threshold,
2089    }
2090}
2091
2092/// Phase 3b of tick: spawn agents for all status:routed tasks up to the parallel limit.
2093#[expect(
2094    clippy::too_many_arguments,
2095    reason = "each argument is engine state the tick passes down"
2096)]
2097#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
2098pub(crate) async fn tick_dispatch_tasks(
2099    backend: &Arc<dyn ExternalBackend>,
2100    tmux: &Arc<TmuxManager>,
2101    repo: &str,
2102    runner: &Arc<TaskRunner>,
2103    capture: &Arc<CaptureService>,
2104    semaphore: &Arc<Semaphore>,
2105    task_manager: &Arc<TaskManager>,
2106    weight_tx: &mpsc::Sender<WeightSignal>,
2107    dispatch_mode: DispatchMode,
2108    dispatching: &Arc<DashMap<String, String>>,
2109    store: &Arc<TaskStore>,
2110    session_map: &HashMap<String, bool>,
2111) -> anyhow::Result<()> {
2112    // Phase 3a already drops no-agent and owner:human, this filter stays as defense in depth
2113    let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
2114
2115    // Also include internal tasks in Routed status.
2116    let internal_routed = task_manager
2117        .list_internal_by_status(DbStatus::Routed)
2118        .await?;
2119    routed_tasks.extend(internal_routed);
2120
2121    // Filter and collect owned tasks to avoid lifetime issues
2122    let dispatchable: Vec<ExternalTask> = routed_tasks
2123        .into_iter()
2124        .filter(|t| !skips_agent(&t.labels))
2125        .collect();
2126
2127    dispatch_tasks_for_repo(
2128        backend,
2129        tmux,
2130        repo,
2131        runner,
2132        capture,
2133        semaphore,
2134        task_manager,
2135        weight_tx,
2136        dispatch_mode,
2137        dispatching,
2138        store,
2139        session_map,
2140        dispatchable,
2141    )
2142    .await
2143}
2144
2145/// Dispatch a pre-fetched list of routed tasks for a single repo.
2146///
2147/// Shared by `tick_dispatch_tasks` (active-repo, per-tick dispatch) and
2148/// `dispatch_routed_tasks_global` (inactive-repo sweep) so both paths spawn
2149/// agents through the exact same worktree/tmux/runner logic.
2150#[expect(
2151    clippy::too_many_arguments,
2152    reason = "each argument is engine state the tick passes down"
2153)]
2154async fn dispatch_tasks_for_repo(
2155    backend: &Arc<dyn ExternalBackend>,
2156    tmux: &Arc<TmuxManager>,
2157    repo: &str,
2158    runner: &Arc<TaskRunner>,
2159    capture: &Arc<CaptureService>,
2160    semaphore: &Arc<Semaphore>,
2161    task_manager: &Arc<TaskManager>,
2162    weight_tx: &mpsc::Sender<WeightSignal>,
2163    dispatch_mode: DispatchMode,
2164    dispatching: &Arc<DashMap<String, String>>,
2165    store: &Arc<TaskStore>,
2166    session_map: &HashMap<String, bool>,
2167    dispatchable: Vec<ExternalTask>,
2168) -> anyhow::Result<()> {
2169    if dispatchable.is_empty() {
2170        tracing::debug!(count = 0, "dispatchable tasks found");
2171        return Ok(());
2172    }
2173
2174    tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2175    if dispatch_mode.is_degraded {
2176        tracing::warn!(
2177            healthy_agents = dispatch_mode.healthy_agents,
2178            threshold = dispatch_mode.threshold,
2179            "degraded mode: using sequential dispatch"
2180        );
2181    }
2182
2183    let sequential_delay = if dispatch_mode.is_degraded {
2184        router_config::sequential_dispatch_delay_ms()
2185    } else {
2186        0
2187    };
2188
2189    let mut ranked: Vec<(Priority, ExternalTask)> = Vec::with_capacity(dispatchable.len());
2190    for task in dispatchable {
2191        let has_pr = match store.resolve_id_by_external(repo, &task.id.0).await {
2192            Ok(Some(id)) => store.get(id).await.is_ok_and(|t| t.pr_number.is_some()),
2193            _ => false,
2194        };
2195        ranked.push((Priority::from_labels(&task.labels).with_pr(has_pr), task));
2196    }
2197    priority::sort_by_priority(&mut ranked, |t| t.0);
2198
2199    for (idx, (priority, task)) in ranked.into_iter().enumerate() {
2200        if dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2201            continue;
2202        }
2203        // In degraded mode, add delay between dispatches to pace the system
2204        if idx > 0 && dispatch_mode.is_degraded {
2205            let delay_ms = sequential_delay;
2206            tracing::debug!(
2207                task_id = task.id.0,
2208                delay_ms = delay_ms,
2209                "sequential dispatch: waiting before dispatch"
2210            );
2211            time::sleep(Duration::from_millis(delay_ms)).await;
2212        }
2213        // In-memory guard: prevents double-dispatch due to GitHub API eventual consistency.
2214        // After update_status(InProgress), the label removal fires a webhook that can
2215        // trigger an immediate tick. GitHub's search index may not yet reflect the label
2216        // change, so list_by_status(Routed) can still return this task. The tmux session
2217        // does not exist until the runner completes worktree setup (~10s later), so the
2218        // session_exists check alone is insufficient.
2219        let dispatch_key = format!("{}/{}", repo, task.id.0);
2220        // Atomically claim the dispatch slot.  DashMap lets us distinguish:
2221        //   - same task already in-flight (expected): log debug and skip.
2222        //   - different task holds the same key (should never happen): log warning and skip.
2223        // A plain DashSet::insert would return false for both cases without distinction.
2224        {
2225            match dispatching.entry(dispatch_key.clone()) {
2226                Entry::Occupied(existing) => {
2227                    let existing_id = existing.get().clone();
2228                    drop(existing); // release shard lock before logging
2229                    if existing_id == task.id.0 {
2230                        tracing::debug!(
2231                            task_id = task.id.0,
2232                            "task already dispatching, skipping duplicate"
2233                        );
2234                    } else {
2235                        tracing::warn!(
2236                            task_id = task.id.0,
2237                            existing_task_id = existing_id,
2238                            dispatch_key,
2239                            "dispatch key collision: unexpected task already holds this key"
2240                        );
2241                    }
2242                    continue;
2243                }
2244                Entry::Vacant(slot) => {
2245                    slot.insert(task.id.0.clone());
2246                    // Promoted from debug! to info! (issue #3526): paired with the
2247                    // "dispatch guard released" line in dispatch_guard.rs, this is the
2248                    // only direct evidence of how long a dispatch guard was actually held
2249                    // for a given key. Both were debug!, silently dropped under the
2250                    // default "orch=info" filter (main.rs) — see the reclaim-skip log
2251                    // promotions above for the full rationale.
2252                    tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2253                }
2254            }
2255        }
2256        // RAII guard — removes dispatch_key on drop even if the spawned task panics.
2257        let dispatch_guard = DispatchGuard::new(Arc::clone(dispatching), dispatch_key.clone());
2258        let task_id = task.id.0.clone();
2259
2260        // Load routing result from store (stored during Phase 3a). Loaded before the
2261        // tmux/semaphore/in_progress steps so a stale cooldown can bail out cheaply.
2262        let route_result = match get_route_result(store, repo, &task_id).await {
2263            Ok(r) => Some(r),
2264            Err(RouteResultError::NoAgent { .. }) => {
2265                tracing::warn!(
2266                    task_id,
2267                    "routed task missing agent — resetting to new for re-routing (#1604)"
2268                );
2269                if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2270                    tracing::error!(task_id, error = %e2, "failed to reset task to new");
2271                }
2272                continue; // dispatch_guard drops here, removing the key
2273            }
2274            Err(e) => {
2275                tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2276                None
2277            }
2278        };
2279
2280        // Re-check the assigned agent/model against the cooldown store: routing and
2281        // dispatch are decoupled by ticks, and a failure recorded after this task was
2282        // routed but before it dispatched would otherwise burn a full run on an
2283        // already-cooled agent/model. Leave the task in Routed so the next tick's
2284        // dispatch pass re-evaluates it — no new per-task defer state (see #3243).
2285        if let Some(route) = route_result.as_ref() {
2286            let agent_cooled = is_agent_in_cooldown(&route.agent);
2287            let model_cooled = route
2288                .model
2289                .as_deref()
2290                .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2291            if agent_cooled || model_cooled {
2292                tracing::warn!(
2293                    task_id,
2294                    agent = route.agent,
2295                    model = route.model.as_deref().unwrap_or(""),
2296                    "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2297                );
2298                continue; // dispatch_guard drops here, removing the key
2299            }
2300        }
2301
2302        // Check if already running (has active session)
2303        let session_name = tmux.session_name(repo, &task_id);
2304        if tmux
2305            .session_blocks_dispatch_from_map(&session_name, session_map)
2306            .await
2307        {
2308            tracing::warn!(
2309                task_id,
2310                session_name,
2311                "task has existing tmux session, skipping dispatch"
2312            );
2313            continue; // dispatch_guard drops here, removing the key
2314        }
2315
2316        // Try to acquire a slot
2317        let Some(permit) = priority::acquire_dispatch_slot(semaphore, priority) else {
2318            tracing::debug!("all parallel slots busy, skipping remaining tasks");
2319            break; // dispatch_guard drops here, removing the key
2320        };
2321
2322        // Mark in_progress BEFORE spawning to prevent double dispatch.
2323        let set_in_progress_result = task_manager
2324            .update_task_status(&task.id, Status::InProgress)
2325            .await;
2326        if let Err(e) = set_in_progress_result {
2327            tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2328            drop(permit);
2329            continue; // dispatch_guard drops here, removing the key
2330        }
2331        tracing::info!(task_id, "dispatching task");
2332
2333        // Register session for capture
2334        capture
2335            .register_session(repo, &task_id, &session_name)
2336            .await;
2337
2338        // Dispatch task
2339        let runner = Arc::clone(runner);
2340        let backend = Arc::clone(backend);
2341        let tmux = Arc::clone(tmux);
2342        let capture = Arc::clone(capture);
2343        let task_id_for_cleanup = task_id.clone();
2344        let task_owned = task;
2345        let weight_tx = weight_tx.clone();
2346        let repo_owned = repo.to_string();
2347        let task_manager_for_spawn = Arc::clone(task_manager);
2348        let store_for_spawn = Arc::clone(store);
2349
2350        let repo_ctx = repo_owned.clone();
2351        tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2352            let _dispatch_guard = dispatch_guard; // released on drop (normal or panic)
2353            let dispatch_start = Instant::now();
2354            match runner
2355                .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2356                .await
2357            {
2358                Ok(signal) => {
2359                    tracing::info!(task_id, "task runner completed");
2360
2361                    let duration = dispatch_start.elapsed().as_secs_f64();
2362
2363                    // Derive a display status from the weight signal.
2364                    // When review agent is enabled, successful tasks go through
2365                    // review before being marked done — UNLESS the response
2366                    // handler already marked the task done (no-op tasks with
2367                    // no PR and no commits should not enter the review cycle).
2368                    let enable_review = config::get("workflow.enable_review_agent")
2369                        .map_or(true, |v| v != "false");
2370                    // Only a successful run can reach the no-PR shortcut, so only then read the PR.
2371                    let has_pr = if enable_review && matches!(signal, WeightSignal::Success { .. })
2372                    {
2373                        match store_for_spawn.resolve_task_id(&repo_owned, &task_id).await {
2374                            Ok(Some(store_id)) => match store_for_spawn.get(store_id).await {
2375                                Ok(t) => t.pr_number.is_some(),
2376                                Err(e) => {
2377                                    tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2378                                    true
2379                                }
2380                            },
2381                            Ok(None) => false,
2382                            Err(e) => {
2383                                tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2384                                true
2385                            }
2386                        }
2387                    } else {
2388                        false
2389                    };
2390                    let display_status = display_status_for(&signal, enable_review, has_pr);
2391
2392                    if display_status == "done" {
2393                        tracing::info!(
2394                            task_id,
2395                            "marking done: the run succeeded with no PR to review"
2396                        );
2397                    }
2398                    tracing::info!(task_id, display_status, "task run finished, applying status");
2399
2400                    // Send weight signal back to the router
2401                    let _ = weight_tx.send(signal).await;
2402
2403                    // All tasks: if needs_review, trigger the review agent.
2404                    // Status updates go through task_manager so internal tasks
2405                    // hit SQLite while external tasks hit GitHub labels.
2406                    if display_status == "needs_review" {
2407                        // Transition to NeedsReview — emits event so the review subscriber
2408                        // fires with the correct duration before the review agent starts.
2409                        if let Err(e) = task_manager_for_spawn
2410                            .update_task_status_with_duration(
2411                                &ExternalId(task_id.clone()),
2412                                Status::NeedsReview,
2413                                Some(duration),
2414                            )
2415                            .await
2416                        {
2417                            tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2418                        }
2419                        // Review agent dispatch is handled by the event-driven subscriber.
2420                    } else {
2421                        // done, blocked, or new (rate-limited): update status directly.
2422                        let final_status = match display_status {
2423                            "done" => Status::Done,
2424                            "blocked" => Status::Blocked,
2425                            _ => Status::New,
2426                        };
2427                        if let Err(e) = task_manager_for_spawn
2428                            .update_task_status_with_duration(
2429                                &ExternalId(task_id.clone()),
2430                                final_status,
2431                                Some(duration),
2432                            )
2433                            .await
2434                        {
2435                            tracing::warn!(task_id, ?e, "failed to update task status after completion");
2436                        }
2437                    }
2438                }
2439                Err(e) => {
2440                    tracing::error!(task_id, ?e, "task runner failed");
2441                    let duration = dispatch_start.elapsed().as_secs_f64();
2442                    if e.downcast_ref::<NoProjectDir>().is_some() {
2443                        // The run failed before any agent started, so nothing exists to review
2444                        block_task_without_project(
2445                            &task_manager_for_spawn,
2446                            &store_for_spawn,
2447                            &repo_owned,
2448                            &task_id,
2449                        )
2450                        .await;
2451                    } else if is_internal_id(&task_id) {
2452                        // Internal tasks have no GitHub issue to comment on.
2453                        if let Err(ue) = task_manager_for_spawn
2454                            .update_task_status_with_duration(
2455                                &ExternalId(task_id.clone()),
2456                                Status::NeedsReview,
2457                                Some(duration),
2458                            )
2459                            .await
2460                        {
2461                            tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2462                        }
2463                    } else {
2464                        // Post comment (best-effort)
2465                        if let Err(comment_err) = backend
2466                            .post_comment(
2467                                &ExternalId(task_id.clone()),
2468                                &format!(
2469                                    "[{}] error: task runner failed: {e}{}",
2470                                    chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2471                                    engine::orch_footer()
2472                                ),
2473                            )
2474                            .await
2475                        {
2476                            tracing::warn!(
2477                                task_id,
2478                                ?comment_err,
2479                                "failed to post error comment to GitHub"
2480                            );
2481                        }
2482                        // Update status immediately to NeedsReview so external tasks
2483                        // don't remain stuck in InProgress until the no-session timeout.
2484                        // The notify subscriber will send the channel notification with duration.
2485                        if let Err(ue) = task_manager_for_spawn
2486                            .update_task_status_with_duration(
2487                                &ExternalId(task_id.clone()),
2488                                Status::NeedsReview,
2489                                Some(duration),
2490                            )
2491                            .await
2492                        {
2493                            tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2494                        }
2495                    }
2496                }
2497            }
2498
2499            // Unregister session from capture
2500            capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2501
2502            // Release the semaphore permit
2503            drop(permit);
2504            // _dispatch_guard drops here, removing the key from the dispatching set.
2505        }));
2506    }
2507    Ok(())
2508}
2509
2510/// Block a task whose repo has no project directory. Nothing exists to review, so
2511/// `NeedsReview` would only feed the refire and rebroadcast loop (#4024)
2512async fn block_task_without_project(
2513    task_manager: &Arc<TaskManager>,
2514    store: &Arc<TaskStore>,
2515    repo: &str,
2516    task_id: &str,
2517) {
2518    let store_id = match store.resolve_task_id(repo, task_id).await {
2519        Ok(Some(id)) => id,
2520        Ok(None) => {
2521            tracing::error!(
2522                task_id,
2523                repo,
2524                "cannot block task without project: not in the store"
2525            );
2526            return;
2527        }
2528        Err(e) => {
2529            tracing::error!(task_id, repo, err = %e, "cannot block task without project: store lookup failed");
2530            return;
2531        }
2532    };
2533    let fields = [
2534        (
2535            "block_reason",
2536            serde_json::json!(NoProjectDir::BLOCK_REASON),
2537        ),
2538        (
2539            "last_error",
2540            serde_json::json!(format!("no project directory for repo {repo}")),
2541        ),
2542    ];
2543    if let Err(e) = task_manager
2544        .update_task_status_and_result_by_store_id(
2545            store_id,
2546            repo,
2547            &ExternalId(task_id.to_string()),
2548            Status::Blocked,
2549            &fields,
2550        )
2551        .await
2552    {
2553        tracing::error!(task_id, repo, err = %e, "failed to block task without project");
2554    }
2555}
2556
2557fn skips_agent(labels: &[String]) -> bool {
2558    labels
2559        .iter()
2560        .any(|label| label == "no-agent" || label == "owner:human")
2561}
2562
2563/// Filter routed tasks down to those belonging to a repo outside `active_repos`,
2564/// drop `no-agent` and `owner:human` tasks, and group the rest by repo.
2565///
2566/// Pure/side-effect-free so it can be unit tested without touching real tmux
2567/// sessions or spawning a runner — see `dispatch_routed_tasks_global`, which is
2568/// the only caller and does the actual (side-effectful) dispatch.
2569fn group_orphaned_routed_tasks(
2570    all_routed: Vec<Task>,
2571    active_repos: &HashSet<String>,
2572) -> HashMap<String, Vec<Task>> {
2573    let mut by_repo: HashMap<String, Vec<Task>> = HashMap::new();
2574    for task in all_routed
2575        .into_iter()
2576        .filter(|t| !active_repos.contains(&t.repo))
2577        .filter(|t| !skips_agent(&t.labels))
2578    {
2579        by_repo.entry(task.repo.clone()).or_default().push(task);
2580    }
2581    by_repo
2582}
2583
2584/// Global dispatch sweep: dispatch status:routed tasks from inactive or removed repos.
2585///
2586/// `tick_dispatch_tasks` is scoped to the active repo per-tick (its `task_manager`
2587/// only queries that repo's tasks) and never processes tasks whose `repo` field
2588/// differs from the current project. When a project is removed from the orch
2589/// config, tasks that reach `routed` for it — including ones routed by the
2590/// `route_new_tasks_global` sweep — have no code path left to dispatch them, so
2591/// they stay `routed` forever. This sweep covers all repos so orphaned routed
2592/// tasks are eventually dispatched.
2593///
2594/// `active_repos` must be the full set of currently-configured project repos,
2595/// not just one project's repo; passing only that project's own repo would make
2596/// every other active repo's routed tasks look "orphaned" too and dispatch them
2597/// a second time through this fallback path, racing the real per-repo dispatch.
2598///
2599/// `TaskRunner::resolve_project_dir` already falls back to the deterministic
2600/// `~/.orch/projects/<owner>/<repo>.git` bare-clone convention when a repo is
2601/// not listed in the active config, so dispatch works without a `ProjectEngine`.
2602#[expect(
2603    clippy::too_many_arguments,
2604    reason = "each argument is engine state the tick passes down"
2605)]
2606pub(crate) async fn dispatch_routed_tasks_global(
2607    tmux: &Arc<TmuxManager>,
2608    capture: &Arc<CaptureService>,
2609    semaphore: &Arc<Semaphore>,
2610    weight_tx: &mpsc::Sender<WeightSignal>,
2611    dispatch_mode: DispatchMode,
2612    dispatching: &Arc<DashMap<String, String>>,
2613    store: &Arc<TaskStore>,
2614    session_map: &HashMap<String, bool>,
2615    active_repos: &HashSet<String>,
2616) -> anyhow::Result<()> {
2617    let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2618    let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2619
2620    if by_repo.is_empty() {
2621        return Ok(());
2622    }
2623
2624    tracing::info!(
2625        repos = by_repo.len(),
2626        "global dispatch sweep: found routed tasks from inactive repos"
2627    );
2628
2629    for (repo, tasks) in by_repo {
2630        let backend: Arc<dyn ExternalBackend> = match GitHubBackend::new(repo.clone()) {
2631            Ok(b) => Arc::new(b),
2632            Err(e) => {
2633                tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2634                continue;
2635            }
2636        };
2637        let task_manager = Arc::new(TaskManager::with_store(
2638            Arc::clone(&backend),
2639            Arc::clone(store),
2640            repo.clone(),
2641        ));
2642        let runner = Arc::new(TaskRunner::new(repo.clone()).with_store(Arc::clone(store)));
2643
2644        // Without a project directory the runner bails before any agent starts, so
2645        // block now instead of taking a dispatch slot first
2646        if let Err(e) = runner.resolve_project_dir().await
2647            && e.downcast_ref::<NoProjectDir>().is_some()
2648        {
2649            tracing::warn!(repo = %repo, count = tasks.len(), "global sweep: repo has no project directory, blocking its routed tasks");
2650            for task in &tasks {
2651                let ext = task
2652                    .external_id
2653                    .clone()
2654                    .unwrap_or_else(|| format!("internal:{}", task.id));
2655                block_task_without_project(&task_manager, store, &repo, &ext).await;
2656            }
2657            continue;
2658        }
2659
2660        let dispatchable: Vec<ExternalTask> =
2661            tasks.iter().map(tasks::store_task_to_external).collect();
2662
2663        if let Err(e) = dispatch_tasks_for_repo(
2664            &backend,
2665            tmux,
2666            &repo,
2667            &runner,
2668            capture,
2669            semaphore,
2670            &task_manager,
2671            weight_tx,
2672            dispatch_mode,
2673            dispatching,
2674            store,
2675            session_map,
2676            dispatchable,
2677        )
2678        .await
2679        {
2680            tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2681        }
2682    }
2683
2684    Ok(())
2685}
2686
2687/// Phase 4 of tick: unblock parent tasks whose sub-issues are all done.
2688pub(crate) async fn tick_unblock_parents(
2689    backend: &Arc<dyn ExternalBackend>,
2690    task_manager: &Arc<TaskManager>,
2691    store: &Arc<TaskStore>,
2692    repo: &str,
2693) -> anyhow::Result<()> {
2694    let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2695
2696    // Pre-filter: skip tasks that are blocked for a known reason (e.g. CI failure,
2697    // escalation). These are not waiting on children, so there is no point making
2698    // a GraphQL call for them.
2699    let store_blocked = store.list_by_status(repo, TaskStatus::Blocked).await?;
2700    let has_block_reason: HashSet<String> = store_blocked
2701        .iter()
2702        .filter(|t| t.block_reason.is_some())
2703        .filter_map(|t| t.external_id.clone())
2704        .collect();
2705
2706    // Spawn a concurrent task for each candidate blocked task.
2707    // Previously these ran sequentially; get_sub_issues is a GraphQL call taking 1-3 s each,
2708    // and with 4 blocked tasks that was 4-12 s of sequential latency every tick.
2709    let handles: Vec<_> = blocked
2710        .into_iter()
2711        .filter(|task| {
2712            if has_block_reason.contains(&task.id.0) {
2713                tracing::debug!(
2714                    task_id = task.id.0,
2715                    "skipping blocked task with known block_reason"
2716                );
2717                false
2718            } else {
2719                true
2720            }
2721        })
2722        .map(|task| {
2723            let backend_clone = Arc::clone(backend);
2724            let task_manager_clone = Arc::clone(task_manager);
2725            let store_clone = Arc::clone(store);
2726            let repo_owned = repo.to_string();
2727            tokio::spawn(async move {
2728                let task_id = &task.id;
2729
2730                let mut children: Vec<ExternalId> = Vec::new();
2731                let mut child_statuses: HashMap<
2732                    String,
2733                    TaskStatus,
2734                > = HashMap::new();
2735
2736                match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2737                    Ok(Some(parent_store_id)) => {
2738                        match store_clone.list_children(&repo_owned, parent_store_id).await {
2739                            Ok(store_children) => {
2740                                for child in store_children {
2741                                    let Some(child_external_id) = child.external_id else {
2742                                        continue;
2743                                    };
2744                                    child_statuses.insert(child_external_id.clone(), child.status);
2745                                    children.push(ExternalId(child_external_id));
2746                                }
2747                            }
2748                            Err(e) => {
2749                                tracing::debug!(
2750                                    task_id = task_id.0,
2751                                    ?e,
2752                                    "failed to list store-linked children"
2753                                );
2754                            }
2755                        }
2756                    }
2757                    Ok(None) => {}
2758                    Err(e) => {
2759                        tracing::debug!(
2760                            task_id = task_id.0,
2761                            ?e,
2762                            "failed to resolve blocked task in store for child lookup"
2763                        );
2764                    }
2765                }
2766
2767                if !is_internal_id(&task_id.0) {
2768                    match backend_clone.get_sub_issues(task_id).await {
2769                        Ok(ids) => {
2770                            let mut seen: HashSet<String> =
2771                                children.iter().map(|child| child.0.clone()).collect();
2772                            for id in ids {
2773                                if seen.insert(id.0.clone()) {
2774                                    children.push(id);
2775                                }
2776                            }
2777                        }
2778                        Err(e) => {
2779                            tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2780                        }
2781                    }
2782                }
2783
2784                // No children means nothing to wait on — skip (may be blocked for other reasons)
2785                if children.is_empty() {
2786                    return;
2787                }
2788
2789                // Check if every child is done using the local store (batched) and
2790                // fall back to the backend only for children not present in the store.
2791                let mut all_done = true;
2792                // Build owned strings for the batch lookup (needed for &str slices below)
2793                let child_ext_strs: Vec<String> =
2794                    children.iter().map(|c| c.0.clone()).collect();
2795                let child_exts: Vec<&str> =
2796                    child_ext_strs.iter().map(String::as_str).collect();
2797                // Query store for statuses of any children present locally
2798                match store_clone
2799                    .get_statuses_by_external_ids(&repo_owned, &child_exts)
2800                    .await
2801                { Ok(map) => {
2802                    for (k, v) in map {
2803                        child_statuses.insert(k, v);
2804                    }
2805                } _ => {
2806                    tracing::debug!(
2807                        task_id = task_id.0,
2808                        "failed to batch-query store for child statuses; falling back to per-child lookups"
2809                    );
2810                }}
2811
2812                for child_id in &children {
2813                    // Check store first
2814                    if let Some(status) = child_statuses.get(&child_id.0) {
2815                        if *status != TaskStatus::Done {
2816                            all_done = false;
2817                            break;
2818                        }
2819                        continue;
2820                    }
2821
2822                    // Not found in store — fall back to backend check
2823                    match backend_clone.get_task(child_id).await {
2824                        Ok(child) => {
2825                            if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2826                                all_done = false;
2827                                break;
2828                            }
2829                        }
2830                        Err(e) => {
2831                            tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2832                            all_done = false;
2833                            break;
2834                        }
2835                    }
2836                }
2837
2838                if all_done {
2839                    tracing::info!(
2840                        task_id = task_id.0,
2841                        children = children.len(),
2842                        "all children done, unblocking parent"
2843                    );
2844                    if let Err(e) =
2845                        task_manager_clone.update_task_status(task_id, Status::New).await
2846                    {
2847                        tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2848                    }
2849                }
2850            })
2851        })
2852        .collect();
2853
2854    for handle in handles {
2855        if let Err(e) = handle.await {
2856            tracing::warn!(?e, "phase 4 sub-issue check panicked");
2857        }
2858    }
2859    Ok(())
2860}
2861
2862/// Phase 5 of tick: run cron job matching and fire any due jobs.
2863pub(crate) async fn tick_job_scheduler(
2864    jobs_path: &Path,
2865    backend: &Arc<dyn ExternalBackend>,
2866    store: Option<&Arc<TaskStore>>,
2867    repo: &str,
2868    transport: Option<&Arc<Transport>>,
2869) -> anyhow::Result<()> {
2870    jobs::tick(jobs_path, backend, store, repo, transport).await
2871}
2872
2873/// Core tick for one project, up to the routing phase.
2874///
2875/// Every phase here is scoped to one project, so all projects can run this at the same time:
2876/// 1. `tick_check_session_completions` — poll tmux for finished sessions
2877/// 2. `tick_detect_silent_agents`      — cooldown agents that produced no output
2878/// 3. `tick_recover_stuck_tasks`       — reset in_progress/in_review tasks with no session
2879///
2880/// The store-wide work that used to sit in here runs once per tick in `tick_all`:
2881/// `tick_global_sweeps` and `reconcile_dependencies`.
2882#[expect(
2883    clippy::too_many_arguments,
2884    reason = "each argument is engine state the tick passes down"
2885)]
2886pub(crate) async fn tick_before_routing(
2887    backend: &Arc<dyn ExternalBackend>,
2888    tmux: &Arc<TmuxManager>,
2889    repo: &str,
2890    capture: &Arc<CaptureService>,
2891    config: &EngineConfig,
2892    task_manager: &Arc<TaskManager>,
2893    dispatching: &Arc<DashMap<String, String>>,
2894    store: &Arc<TaskStore>,
2895    session_map: &HashMap<String, bool>,
2896) {
2897    let mut phases: Vec<(&str, u128)> = Vec::new();
2898    let mut at = Instant::now();
2899
2900    if let Err(e) = tick_check_session_completions(tmux, repo, capture, store).await {
2901        tracing::warn!(err = %e, "session completion phase failed");
2902    }
2903    phases.push(("sessions", at.elapsed().as_millis()));
2904
2905    // Every node tends its own sessions, only the leader decides. Followers dispatch once
2906    // tasks are assigned to nodes
2907    if !store.is_leader() {
2908        super::log_phase_timings("tick.pre_routing", repo, &phases);
2909        return;
2910    }
2911
2912    at = Instant::now();
2913    if let Err(e) =
2914        tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await
2915    {
2916        tracing::warn!(err = %e, "silent-agent detection phase failed");
2917    }
2918    phases.push(("silent_agents", at.elapsed().as_millis()));
2919
2920    at = Instant::now();
2921    if let Err(e) = tick_recover_stuck_tasks(
2922        backend,
2923        tmux,
2924        repo,
2925        task_manager,
2926        config,
2927        store,
2928        session_map,
2929        dispatching,
2930    )
2931    .await
2932    {
2933        tracing::warn!(err = %e, "stuck-task recovery phase failed");
2934    }
2935    phases.push(("stuck_tasks", at.elapsed().as_millis()));
2936
2937    super::log_phase_timings("tick.pre_routing", repo, &phases);
2938}
2939
2940/// Core tick for one project, after the routing phase.
2941///
2942/// Takes the router's decision as a `DispatchMode` value instead of the router itself, so
2943/// every project can run this at the same time:
2944/// 1. `tick_dispatch_tasks`  — spawn agents for status:routed tasks
2945/// 2. `tick_unblock_parents` — unblock parents whose sub-issues are all done
2946/// 3. `tick_job_scheduler`   — fire due cron jobs
2947#[expect(
2948    clippy::too_many_arguments,
2949    reason = "each argument is engine state the tick passes down"
2950)]
2951pub(crate) async fn tick_after_routing(
2952    backend: &Arc<dyn ExternalBackend>,
2953    tmux: &Arc<TmuxManager>,
2954    repo: &str,
2955    runner: &Arc<TaskRunner>,
2956    capture: &Arc<CaptureService>,
2957    semaphore: &Arc<Semaphore>,
2958    project_dir: &Path,
2959    task_manager: &Arc<TaskManager>,
2960    weight_tx: &mpsc::Sender<WeightSignal>,
2961    dispatch_mode: DispatchMode,
2962    dispatching: &Arc<DashMap<String, String>>,
2963    store: &Arc<TaskStore>,
2964    transport: Option<&Arc<Transport>>,
2965    session_map: &HashMap<String, bool>,
2966) {
2967    let mut phases: Vec<(&str, u128)> = Vec::new();
2968    let mut at = Instant::now();
2969
2970    if let Err(e) = tick_dispatch_tasks(
2971        backend,
2972        tmux,
2973        repo,
2974        runner,
2975        capture,
2976        semaphore,
2977        task_manager,
2978        weight_tx,
2979        dispatch_mode,
2980        dispatching,
2981        store,
2982        session_map,
2983    )
2984    .await
2985    {
2986        tracing::warn!(err = %e, "dispatch phase failed");
2987    }
2988    phases.push(("dispatch", at.elapsed().as_millis()));
2989
2990    at = Instant::now();
2991    if let Err(e) = tick_unblock_parents(backend, task_manager, store, repo).await {
2992        tracing::warn!(err = %e, "parent-unblock phase failed");
2993    }
2994    phases.push(("unblock_parents", at.elapsed().as_millis()));
2995
2996    at = Instant::now();
2997    let jobs_path = project_dir.join(".orch.yml");
2998    if let Err(e) = tick_job_scheduler(&jobs_path, backend, Some(store), repo, transport).await {
2999        tracing::error!(?e, "job scheduler tick failed");
3000    }
3001    phases.push(("jobs", at.elapsed().as_millis()));
3002
3003    super::log_phase_timings("tick.post_routing", repo, &phases);
3004}
3005
3006/// Core tick — runs every 10s for every project.
3007///
3008/// Projects no longer wait on each other: the phases around routing run for all of them at
3009/// once, so tick time tracks the slowest project instead of the sum. Routing stays serial
3010/// because it needs the router write lock and one shared `RoutingTickQuota`, which is what
3011/// gives `router.max_tasks_per_tick` its meaning.
3012///
3013/// The tmux session map, the global sweeps and the dependency reconcile are the same for
3014/// every project, so they run once per tick rather than once per project.
3015#[expect(
3016    clippy::too_many_arguments,
3017    reason = "each argument is engine state the tick passes down"
3018)]
3019pub(crate) async fn tick_all(
3020    engines: &[ProjectEngine],
3021    tmux: &Arc<TmuxManager>,
3022    capture: &Arc<CaptureService>,
3023    semaphore: &Arc<Semaphore>,
3024    config: &EngineConfig,
3025    router: &Arc<RwLock<Router>>,
3026    weight_tx: &mpsc::Sender<WeightSignal>,
3027    dispatching: &Arc<DashMap<String, String>>,
3028    transport: Option<&Arc<Transport>>,
3029    active_repos: &HashSet<String>,
3030    auto_merge_in_flight: &Arc<DashSet<String>>,
3031) {
3032    let _tick_span = tracing::info_span!("engine.tick").entered();
3033
3034    let Some(first) = engines.first() else {
3035        return;
3036    };
3037    // All project engines share one SQLite store, so leadership and the global sweeps below
3038    // are the same whichever project they are read from
3039    let store = &first.store;
3040
3041    // Calls made since the previous tick, so hourly quota exhaustion can be traced
3042    for (project, call_type, calls) in http::drain_call_counts().into_iter().take(10) {
3043        tracing::debug!(
3044            project,
3045            call_type,
3046            calls,
3047            "github api calls since last tick"
3048        );
3049    }
3050
3051    // One `tmux list-panes -a` for every project's stuck-task recovery, instead of one each
3052    let session_map = tmux.batch_session_active().await;
3053
3054    let mut passes = Vec::with_capacity(engines.len());
3055    for engine in engines {
3056        passes.push(REPO_CONTEXT.scope(
3057            engine.repo.clone(),
3058            tick_before_routing(
3059                &engine.backend,
3060                tmux,
3061                &engine.repo,
3062                capture,
3063                config,
3064                &engine.task_manager,
3065                dispatching,
3066                &engine.store,
3067                &session_map,
3068            ),
3069        ));
3070    }
3071    future::join_all(passes).await;
3072
3073    if !store.is_leader() {
3074        return;
3075    }
3076
3077    // Store-wide, so once per tick rather than once per project. Both read every repo, and
3078    // the NeedsReview sweep increments a counter it just read, which concurrent per-project
3079    // passes would each consume (#3864)
3080    let at = Instant::now();
3081    {
3082        let router_guard = router.read().await;
3083        tick_global_sweeps(
3084            &first.task_manager,
3085            store,
3086            &router_guard,
3087            dispatching,
3088            auto_merge_in_flight,
3089            active_repos,
3090        )
3091        .await;
3092    }
3093    let sweeps_ms = at.elapsed().as_millis();
3094
3095    let at = Instant::now();
3096    recover_untracked_repo_stuck_tasks(
3097        tmux,
3098        config,
3099        store,
3100        &session_map,
3101        dispatching,
3102        active_repos,
3103    )
3104    .await;
3105    let untracked_ms = at.elapsed().as_millis();
3106
3107    let at = Instant::now();
3108    if let Err(err) = dependencies::reconcile_dependencies(
3109        first.backend.as_ref(),
3110        store,
3111        transport.map(AsRef::as_ref),
3112    )
3113    .await
3114    {
3115        tracing::warn!(err = %err, "dependency reconcile failed");
3116    }
3117    super::log_phase_timings(
3118        "tick.global",
3119        &first.repo,
3120        &[
3121            ("global_sweeps", sweeps_ms),
3122            ("untracked_stuck", untracked_ms),
3123            ("dependencies", at.elapsed().as_millis()),
3124        ],
3125    );
3126
3127    {
3128        let mut router_guard = router.write().await;
3129        let mut routing_quota = RoutingTickQuota::new(router_config::max_tasks_per_routing_tick());
3130        for engine in engines {
3131            let at = Instant::now();
3132            REPO_CONTEXT
3133                .scope(engine.repo.clone(), async {
3134                    if let Err(e) = tick_route_tasks(
3135                        &engine.backend,
3136                        &engine.task_manager,
3137                        &mut router_guard,
3138                        &engine.store,
3139                        &engine.repo,
3140                        &mut routing_quota,
3141                    )
3142                    .await
3143                    {
3144                        tracing::warn!(err = %e, "routing phase failed");
3145                    }
3146                })
3147                .await;
3148            super::log_phase_timings(
3149                "tick.routing",
3150                &engine.repo,
3151                &[("route", at.elapsed().as_millis())],
3152            );
3153        }
3154
3155        // Global routing sweep for tasks from inactive or removed repos.
3156        // tick_route_tasks is scoped to the active repo; tasks from projects no longer
3157        // in config are never returned by list_routable and would stay in 'new' indefinitely.
3158        if let Err(e) =
3159            route_new_tasks_global(&mut router_guard, store, active_repos, &mut routing_quota).await
3160        {
3161            tracing::warn!(err = %e, "global new-task routing sweep failed");
3162        }
3163    }
3164
3165    let dispatch_mode = dispatch_mode_from_router(&*router.read().await);
3166
3167    // Re-read the session map: the completion phase above kills the sessions of finished
3168    // agents, and dispatch must not treat a killed session as one that still blocks a task
3169    let session_map = tmux.batch_session_active().await;
3170
3171    let mut passes = Vec::with_capacity(engines.len());
3172    for engine in engines {
3173        passes.push(REPO_CONTEXT.scope(
3174            engine.repo.clone(),
3175            tick_after_routing(
3176                &engine.backend,
3177                tmux,
3178                &engine.repo,
3179                &engine.runner,
3180                capture,
3181                semaphore,
3182                &engine.project_dir,
3183                &engine.task_manager,
3184                weight_tx,
3185                dispatch_mode,
3186                dispatching,
3187                &engine.store,
3188                transport,
3189                &session_map,
3190            ),
3191        ));
3192    }
3193    future::join_all(passes).await;
3194
3195    // Global dispatch sweep for routed tasks from inactive or removed repos.
3196    // tick_dispatch_tasks is scoped to each active repo; tasks from projects no longer
3197    // in config are never returned by list_external_by_status and would stay in
3198    // 'routed' indefinitely (see #3413). Excludes the full active-repo set, not just
3199    // one repo, so it doesn't treat other active repos as orphaned.
3200    if let Err(e) = dispatch_routed_tasks_global(
3201        tmux,
3202        capture,
3203        semaphore,
3204        weight_tx,
3205        dispatch_mode,
3206        dispatching,
3207        store,
3208        &session_map,
3209        active_repos,
3210    )
3211    .await
3212    {
3213        tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
3214    }
3215}
3216
3217/// Status a finished run moves to. Only a successful run with no PR (or with the review agent
3218/// off) becomes `done`. A failed run stays in `needs_review` with its issue open
3219pub(crate) fn display_status_for(
3220    signal: &WeightSignal,
3221    enable_review: bool,
3222    has_pr: bool,
3223) -> &'static str {
3224    match signal {
3225        WeightSignal::Success { .. } if enable_review => {
3226            if has_pr {
3227                "needs_review"
3228            } else {
3229                "done"
3230            }
3231        }
3232        // Silence detection or another non-rate-limit reset already moved the task to "new".
3233        // Converting it to needs_review would trigger the no-code shortcut and falsely close it
3234        WeightSignal::RateLimited { .. } | WeightSignal::Rerouted => "new",
3235        WeightSignal::Blocked => "blocked",
3236        // The review path re-routes a task with no PR and no commits
3237        WeightSignal::Failed { .. } => "needs_review",
3238        // None means the runner guard skipped the task or the status was unrecognised
3239        WeightSignal::None if enable_review => "needs_review",
3240        WeightSignal::Success { .. } | WeightSignal::None => "done",
3241    }
3242}
3243
3244#[cfg(test)]
3245mod tests {
3246    use super::*;
3247    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3248    use crate::channels::transport::Transport;
3249    use crate::engine::RunTimeouts;
3250    use crate::engine::cooldown;
3251    use crate::engine::router::Router;
3252    use crate::engine::router::RouterConfig;
3253    use crate::engine::tasks::TaskManager;
3254    use crate::store::DepSource;
3255    use crate::store::NewDependency;
3256    use crate::store::NewTask;
3257    use crate::store::StoreRoute;
3258    use crate::store::TaskStatus as DbStatus;
3259    use crate::store::TaskStatus;
3260    use crate::store::TaskStore;
3261    use crate::store::UpsertExternal;
3262    use crate::tmux::{SessionGuard, TmuxManager};
3263    use async_trait::async_trait;
3264    use dashmap::mapref::entry::Entry;
3265    use std::cell::Cell;
3266    use std::fs;
3267    use std::path::PathBuf;
3268    use std::sync::{Arc, Mutex};
3269    use tokio::sync::RwLock;
3270    use tokio::sync::oneshot;
3271    use tokio::task;
3272
3273    #[test]
3274    fn timed_out_run_without_pr_never_becomes_done() {
3275        let failed = WeightSignal::Failed {
3276            agent: "claude".to_string(),
3277        };
3278        assert_eq!(display_status_for(&failed, true, false), "needs_review");
3279        assert_eq!(display_status_for(&failed, false, false), "needs_review");
3280    }
3281
3282    #[test]
3283    fn done_needs_a_successful_run_with_no_pr() {
3284        let ok = WeightSignal::Success {
3285            agent: "claude".to_string(),
3286        };
3287        assert_eq!(display_status_for(&ok, true, true), "needs_review");
3288        assert_eq!(display_status_for(&ok, true, false), "done");
3289        assert_eq!(
3290            display_status_for(&WeightSignal::Rerouted, true, false),
3291            "new"
3292        );
3293        assert_eq!(
3294            display_status_for(&WeightSignal::None, true, false),
3295            "needs_review"
3296        );
3297    }
3298    #[test]
3299    fn skips_agent_for_human_and_no_agent_labels() {
3300        assert!(skips_agent(&["owner:human".to_string()]));
3301        assert!(skips_agent(&["no-agent".to_string()]));
3302        assert!(!skips_agent(&["status:new".to_string()]));
3303    }
3304
3305    /// A router reporting one available agent with no healthy "review" model, so the
3306    /// global NeedsReview refire sweep in `tick_global_sweeps` treats every review agent
3307    /// as cooled and skips — keeping it a no-op for tests unrelated to it.
3308    fn cooled_review_router_for_test() -> Router {
3309        Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()])
3310    }
3311
3312    // ── minimal mock backend ─────────────────────────────────────────────────
3313
3314    /// Configurable mock backend for tick tests.
3315    struct MockBackend {
3316        /// Tasks returned by `list_by_status(Blocked)`.
3317        blocked_tasks: Vec<ExternalTask>,
3318        /// Sub-issues map: task_id → list of child ExternalIds.
3319        sub_issues: HashMap<String, Vec<ExternalId>>,
3320        /// Tasks returned by `get_task`. Keyed by id.0.
3321        tasks_by_id: HashMap<String, ExternalTask>,
3322        /// Recorded `get_sub_issues` calls.
3323        get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
3324        /// Recorded `update_status` calls.
3325        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
3326        /// Recorded `remove_label` calls.
3327        removed_labels: Arc<Mutex<Vec<(String, String)>>>,
3328        /// Recorded `post_comment` calls.
3329        posted_comments: Arc<Mutex<Vec<(String, String)>>>,
3330    }
3331
3332    impl MockBackend {
3333        fn new() -> Self {
3334            Self {
3335                blocked_tasks: vec![],
3336                sub_issues: HashMap::new(),
3337                tasks_by_id: HashMap::new(),
3338                get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
3339                status_updates: Arc::new(Mutex::new(vec![])),
3340                removed_labels: Arc::new(Mutex::new(vec![])),
3341                posted_comments: Arc::new(Mutex::new(vec![])),
3342            }
3343        }
3344    }
3345
3346    fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
3347        ExternalTask {
3348            id: ExternalId(id.to_string()),
3349            title: format!("Task {id}"),
3350            body: String::new(),
3351            state: "open".to_string(),
3352            labels: labels.iter().map(ToString::to_string).collect(),
3353            author: "bot".to_string(),
3354            created_at: "2026-01-01T00:00:00Z".to_string(),
3355            updated_at: "2026-01-01T00:00:00Z".to_string(),
3356            url: format!("https://github.com/test/test/issues/{id}"),
3357        }
3358    }
3359
3360    #[async_trait]
3361    impl ExternalBackend for MockBackend {
3362        fn name(&self) -> &'static str {
3363            "mock"
3364        }
3365        async fn create_task(
3366            &self,
3367            _title: &str,
3368            _body: &str,
3369            _labels: &[String],
3370        ) -> anyhow::Result<ExternalId> {
3371            Ok(ExternalId("new".to_string()))
3372        }
3373        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3374            self.tasks_by_id
3375                .get(&id.0)
3376                .cloned()
3377                .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
3378        }
3379        async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3380            if status == Status::Blocked {
3381                Ok(self.blocked_tasks.clone())
3382            } else {
3383                Ok(vec![])
3384            }
3385        }
3386        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3387            Ok(vec![])
3388        }
3389        async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
3390            self.posted_comments
3391                .lock()
3392                .unwrap()
3393                .push((_id.0.clone(), _body.to_string()));
3394            Ok(())
3395        }
3396        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
3397            Ok(())
3398        }
3399        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
3400            self.removed_labels
3401                .lock()
3402                .unwrap()
3403                .push((_id.0.clone(), _label.to_string()));
3404            Ok(())
3405        }
3406        async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3407            self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
3408            Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
3409        }
3410        async fn create_sub_task(
3411            &self,
3412            _parent: &ExternalId,
3413            _title: &str,
3414            _body: &str,
3415            _labels: &[String],
3416        ) -> anyhow::Result<ExternalId> {
3417            Ok(ExternalId("child".to_string()))
3418        }
3419        async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
3420            Ok(())
3421        }
3422        async fn has_open_issue_with_title(
3423            &self,
3424            _title: &str,
3425            _label: &str,
3426        ) -> anyhow::Result<bool> {
3427            Ok(false)
3428        }
3429        async fn health_check(&self) -> anyhow::Result<()> {
3430            Ok(())
3431        }
3432        async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
3433            Ok(false)
3434        }
3435        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3436            Ok(Some("testbot".to_string()))
3437        }
3438        async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
3439            Ok(vec![])
3440        }
3441        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3442            self.status_updates
3443                .lock()
3444                .unwrap()
3445                .push((id.0.clone(), status));
3446            Ok(())
3447        }
3448    }
3449
3450    fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
3451        Arc::new(TaskManager::new(backend))
3452    }
3453
3454    // ── tick_unblock_parents ─────────────────────────────────────────────────
3455
3456    #[tokio::test]
3457    async fn unblock_parents_unblocks_when_all_children_done() {
3458        let mut mock = MockBackend::new();
3459
3460        // Blocked parent with two done children
3461        let parent = make_task("10", &["status:blocked"]);
3462        mock.blocked_tasks.push(parent.clone());
3463        mock.sub_issues.insert(
3464            "10".to_string(),
3465            vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
3466        );
3467        mock.tasks_by_id
3468            .insert("11".to_string(), make_task("11", &["status:done"]));
3469        mock.tasks_by_id
3470            .insert("12".to_string(), make_task("12", &["status:done"]));
3471
3472        let status_updates = Arc::clone(&mock.status_updates);
3473        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3474        let task_manager = make_task_manager(Arc::clone(&backend));
3475        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3476
3477        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3478            .await
3479            .unwrap();
3480
3481        let updates = status_updates.lock().unwrap();
3482        assert_eq!(updates.len(), 1, "parent should be unblocked");
3483        assert_eq!(updates[0], ("10".to_string(), Status::New));
3484    }
3485
3486    #[tokio::test]
3487    async fn unblock_parents_skips_when_child_not_done() {
3488        let mut mock = MockBackend::new();
3489
3490        let parent = make_task("20", &["status:blocked"]);
3491        mock.blocked_tasks.push(parent.clone());
3492        mock.sub_issues.insert(
3493            "20".to_string(),
3494            vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3495        );
3496        // Child 21 is done, child 22 is still in_progress
3497        mock.tasks_by_id
3498            .insert("21".to_string(), make_task("21", &["status:done"]));
3499        mock.tasks_by_id
3500            .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3501
3502        let status_updates = Arc::clone(&mock.status_updates);
3503        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3504        let task_manager = make_task_manager(Arc::clone(&backend));
3505        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3506
3507        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3508            .await
3509            .unwrap();
3510
3511        let updates = status_updates.lock().unwrap();
3512        assert!(
3513            updates.is_empty(),
3514            "should not unblock when a child is still running"
3515        );
3516    }
3517
3518    #[tokio::test]
3519    async fn unblock_parents_skips_task_with_no_children() {
3520        let mut mock = MockBackend::new();
3521
3522        // Blocked task with no sub-issues (blocked for a different reason)
3523        let parent = make_task("30", &["status:blocked"]);
3524        mock.blocked_tasks.push(parent.clone());
3525        // No entry in sub_issues → get_sub_issues returns []
3526
3527        let status_updates = Arc::clone(&mock.status_updates);
3528        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3529        let task_manager = make_task_manager(Arc::clone(&backend));
3530        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3531
3532        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3533            .await
3534            .unwrap();
3535
3536        let updates = status_updates.lock().unwrap();
3537        assert!(
3538            updates.is_empty(),
3539            "should not unblock tasks with no sub-issues"
3540        );
3541    }
3542
3543    #[tokio::test]
3544    async fn unblock_parents_skips_task_with_block_reason_in_store() {
3545        // A blocked task that has a known block_reason in the store should never
3546        // trigger a get_sub_issues GraphQL call — it is not waiting on children.
3547        let mut mock = MockBackend::new();
3548
3549        // Blocked parent in the backend list
3550        let parent = make_task("60", &["status:blocked"]);
3551        mock.blocked_tasks.push(parent.clone());
3552        // Give it children in the backend so that, if the skip logic is wrong,
3553        // the parent would get unblocked.
3554        mock.sub_issues
3555            .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3556        mock.tasks_by_id
3557            .insert("61".to_string(), make_task("61", &["status:done"]));
3558
3559        let status_updates = Arc::clone(&mock.status_updates);
3560        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3561        let task_manager = make_task_manager(Arc::clone(&backend));
3562
3563        // Insert the blocked task into the store with a block_reason set.
3564        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3565        let task_id = store
3566            .upsert_external(&crate::store::UpsertExternal {
3567                repo: "test/repo",
3568                ext_id: "60",
3569                title: "Task 60",
3570                body: "",
3571                author: "bot",
3572                url: "https://github.com/test/test/issues/60",
3573                labels: &[],
3574                origin: "github",
3575            })
3576            .await
3577            .unwrap();
3578        store
3579            .update_status(task_id, TaskStatus::Blocked)
3580            .await
3581            .unwrap();
3582        store
3583            .set_block_reason(task_id, Some("CI failure limit reached"))
3584            .await
3585            .unwrap();
3586
3587        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3588            .await
3589            .unwrap();
3590
3591        let updates = status_updates.lock().unwrap();
3592        assert!(
3593            updates.is_empty(),
3594            "task with block_reason should be skipped — no API call, no unblock"
3595        );
3596    }
3597
3598    #[tokio::test]
3599    async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3600        let mock = MockBackend::new();
3601        let get_sub_issues_calls = Arc::clone(&mock.get_sub_issues_calls);
3602        let status_updates = Arc::clone(&mock.status_updates);
3603        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3604        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3605        let task_manager = Arc::new(TaskManager::with_store(
3606            Arc::clone(&backend),
3607            Arc::clone(&store),
3608            "test/repo".to_string(),
3609        ));
3610
3611        let parent_id = store
3612            .create_internal("test/repo", "Parent", "", "job", "daily", None)
3613            .await
3614            .unwrap();
3615        store
3616            .update_status(parent_id, TaskStatus::Blocked)
3617            .await
3618            .unwrap();
3619        let child_id = store
3620            .create(&NewTask {
3621                external_id: Some("91".to_string()),
3622                repo: "test/repo".to_string(),
3623                origin: "github".to_string(),
3624                title: "Child".to_string(),
3625                body: String::new(),
3626                source: String::new(),
3627                source_id: String::new(),
3628                author: String::new(),
3629                url: String::new(),
3630                labels: vec![],
3631                parent_id: Some(parent_id),
3632            })
3633            .await
3634            .unwrap();
3635        store
3636            .update_status(child_id, TaskStatus::Done)
3637            .await
3638            .unwrap();
3639
3640        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3641            .await
3642            .unwrap();
3643
3644        let parent = store.get(parent_id).await.unwrap();
3645        assert_eq!(parent.status, TaskStatus::New);
3646        assert!(
3647            status_updates.lock().unwrap().is_empty(),
3648            "internal parent should be updated in store without backend mirroring"
3649        );
3650        assert!(
3651            get_sub_issues_calls.lock().unwrap().is_empty(),
3652            "internal parent should not query backend sub-issues"
3653        );
3654    }
3655
3656    // ── tick_recover_stuck_tasks (InReview) ──────────────────────────────────
3657
3658    /// Set `updated_at` to a far-past date for a task in an in-memory store.
3659    #[cfg(test)]
3660    async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3661        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3662            .bind(task_id)
3663            .execute(store.pool())
3664            .await
3665            .unwrap();
3666    }
3667
3668    /// Age the in_review status-entry the same way the stale sweep tests do
3669    async fn set_in_review_entered_at_past(store: &TaskStore, task_id: i64) {
3670        sqlx::query(
3671            "UPDATE task_activity SET timestamp = '2020-01-01T00:00:00Z'
3672             WHERE task_id = ? AND to_status = 'in_review'",
3673        )
3674        .bind(task_id)
3675        .execute(store.pool())
3676        .await
3677        .unwrap();
3678    }
3679
3680    #[tokio::test]
3681    async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3682        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3683        let mock = MockBackend::new();
3684        let status_updates = Arc::clone(&mock.status_updates);
3685        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3686        let task_manager = Arc::new(TaskManager::with_store(
3687            Arc::clone(&backend),
3688            Arc::clone(&store),
3689            "owner/repo".to_string(),
3690        ));
3691        let tmux = Arc::new(TmuxManager::new());
3692        let config = EngineConfig {
3693            no_session_stuck_timeout: 600,
3694            stuck_timeout: 1800,
3695            ..EngineConfig::default()
3696        };
3697
3698        // Insert an external task and age both the status entry and updated_at
3699        let id = store
3700            .upsert_external(&crate::store::UpsertExternal {
3701                repo: "owner/repo",
3702                ext_id: "99",
3703                title: "Stuck InReview",
3704                body: "",
3705                author: "",
3706                url: "",
3707                labels: &[],
3708                origin: "github",
3709            })
3710            .await
3711            .unwrap();
3712        store.update_status(id, TaskStatus::InReview).await.unwrap();
3713        set_in_review_entered_at_past(&store, id).await;
3714        set_task_updated_at_past(&store, id).await;
3715
3716        tick_recover_stuck_tasks(
3717            &backend,
3718            &tmux,
3719            "owner/repo",
3720            &task_manager,
3721            &config,
3722            &store,
3723            &HashMap::new(),
3724            &Arc::new(DashMap::new()),
3725        )
3726        .await
3727        .unwrap();
3728
3729        let updates = status_updates.lock().unwrap();
3730        assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3731        assert_eq!(updates[0].0, "99");
3732        assert_eq!(updates[0].1, Status::NeedsReview);
3733    }
3734
3735    #[tokio::test]
3736    async fn recover_stuck_tasks_skips_recent_in_review() {
3737        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3738        let mock = MockBackend::new();
3739        let status_updates = Arc::clone(&mock.status_updates);
3740        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3741        let task_manager = Arc::new(TaskManager::with_store(
3742            Arc::clone(&backend),
3743            Arc::clone(&store),
3744            "owner/repo".to_string(),
3745        ));
3746        let tmux = Arc::new(TmuxManager::new());
3747        let config = EngineConfig {
3748            no_session_stuck_timeout: 600, // 10 minutes — recent task won't trigger
3749            stuck_timeout: 1800,
3750            ..EngineConfig::default()
3751        };
3752
3753        // Age updated_at, leave the in_review status entry at now
3754        let id = store
3755            .upsert_external(&crate::store::UpsertExternal {
3756                repo: "owner/repo",
3757                ext_id: "100",
3758                title: "Recent InReview",
3759                body: "",
3760                author: "",
3761                url: "",
3762                labels: &[],
3763                origin: "github",
3764            })
3765            .await
3766            .unwrap();
3767        store.update_status(id, TaskStatus::InReview).await.unwrap();
3768        set_task_updated_at_past(&store, id).await;
3769
3770        tick_recover_stuck_tasks(
3771            &backend,
3772            &tmux,
3773            "owner/repo",
3774            &task_manager,
3775            &config,
3776            &store,
3777            &HashMap::new(),
3778            &Arc::new(DashMap::new()),
3779        )
3780        .await
3781        .unwrap();
3782
3783        let updates = status_updates.lock().unwrap();
3784        assert!(
3785            updates.is_empty(),
3786            "recent InReview task should not be recovered"
3787        );
3788    }
3789
3790    #[tokio::test]
3791    async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3792        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3793        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3794        let task_manager = Arc::new(TaskManager::with_store(
3795            Arc::clone(&backend),
3796            Arc::clone(&store),
3797            "owner/repo".to_string(),
3798        ));
3799        let tmux = Arc::new(TmuxManager::new());
3800        let config = EngineConfig {
3801            no_session_stuck_timeout: 600,
3802            stuck_timeout: 1800,
3803            ..EngineConfig::default()
3804        };
3805
3806        // Create an internal task in InReview and age both clocks
3807        let id = store
3808            .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3809            .await
3810            .unwrap();
3811        store.update_status(id, TaskStatus::InReview).await.unwrap();
3812        set_in_review_entered_at_past(&store, id).await;
3813        set_task_updated_at_past(&store, id).await;
3814
3815        tick_recover_stuck_tasks(
3816            &backend,
3817            &tmux,
3818            "owner/repo",
3819            &task_manager,
3820            &config,
3821            &store,
3822            &HashMap::new(),
3823            &Arc::new(DashMap::new()),
3824        )
3825        .await
3826        .unwrap();
3827
3828        let task = store.get(id).await.unwrap();
3829        assert_eq!(
3830            task.status,
3831            TaskStatus::NeedsReview,
3832            "stuck internal InReview task should be reset to NeedsReview"
3833        );
3834    }
3835
3836    #[tokio::test]
3837    async fn recover_stuck_tasks_ages_in_review_from_status_entry_not_updated_at() {
3838        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3839        let mock = MockBackend::new();
3840        let status_updates = Arc::clone(&mock.status_updates);
3841        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3842        let task_manager = Arc::new(TaskManager::with_store(
3843            Arc::clone(&backend),
3844            Arc::clone(&store),
3845            "owner/repo".to_string(),
3846        ));
3847        let tmux = Arc::new(TmuxManager::new());
3848        let config = EngineConfig {
3849            no_session_stuck_timeout: 600,
3850            stuck_timeout: 1800,
3851            ..EngineConfig::default()
3852        };
3853
3854        let id = store
3855            .upsert_external(&crate::store::UpsertExternal {
3856                repo: "owner/repo",
3857                ext_id: "102",
3858                title: "Fresh updated_at InReview",
3859                body: "",
3860                author: "",
3861                url: "",
3862                labels: &[],
3863                origin: "github",
3864            })
3865            .await
3866            .unwrap();
3867        store.update_status(id, TaskStatus::InReview).await.unwrap();
3868        set_in_review_entered_at_past(&store, id).await;
3869        store
3870            .set_fields(id, &[("pr_number", serde_json::json!(42))])
3871            .await
3872            .unwrap();
3873
3874        tick_recover_stuck_tasks(
3875            &backend,
3876            &tmux,
3877            "owner/repo",
3878            &task_manager,
3879            &config,
3880            &store,
3881            &HashMap::new(),
3882            &Arc::new(DashMap::new()),
3883        )
3884        .await
3885        .unwrap();
3886
3887        let updates = status_updates.lock().unwrap();
3888        assert_eq!(
3889            updates.len(),
3890            1,
3891            "in_review stuck recovery must age from the status entry even when updated_at is fresh"
3892        );
3893        assert_eq!(updates[0].0, "102");
3894        assert_eq!(updates[0].1, Status::NeedsReview);
3895    }
3896
3897    /// An in_review task whose review session is gone goes back to needs_review, unless its
3898    /// review already recorded a decision and the task is only waiting on the merge
3899    async fn orphaned_review_status_after_recovery(with_decision: bool) -> TaskStatus {
3900        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3901        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3902        let task_manager = Arc::new(TaskManager::with_store(
3903            Arc::clone(&backend),
3904            Arc::clone(&store),
3905            "owner/repo".to_string(),
3906        ));
3907        let tmux = Arc::new(TmuxManager::new());
3908
3909        let id = store
3910            .upsert_external(&crate::store::UpsertExternal {
3911                repo: "owner/repo",
3912                ext_id: "55",
3913                title: "Orphaned review",
3914                body: "",
3915                author: "",
3916                url: "",
3917                labels: &[],
3918                origin: "github",
3919            })
3920            .await
3921            .unwrap();
3922        store.update_status(id, TaskStatus::InReview).await.unwrap();
3923        // Entered in_review long ago. updated_at stays fresh, as review_poll rewrites it
3924        set_in_review_entered_at_past(&store, id).await;
3925        if with_decision {
3926            store::store_log_activity(
3927                &Some(Arc::clone(&store)),
3928                "owner/repo",
3929                "55",
3930                "review_decision",
3931                None,
3932                None,
3933                None::<&str>,
3934                None::<&str>,
3935                Some(&serde_json::json!({"decision": "approve"})),
3936            )
3937            .await;
3938        }
3939
3940        tick_recover_stuck_tasks(
3941            &backend,
3942            &tmux,
3943            "owner/repo",
3944            &task_manager,
3945            &EngineConfig::default(),
3946            &store,
3947            &HashMap::new(),
3948            &Arc::new(DashMap::new()),
3949        )
3950        .await
3951        .unwrap();
3952
3953        store.get(id).await.unwrap().status
3954    }
3955
3956    #[tokio::test]
3957    async fn stale_in_review_recovery_resets_orphaned_task_without_session() {
3958        assert_eq!(
3959            orphaned_review_status_after_recovery(false).await,
3960            TaskStatus::NeedsReview
3961        );
3962    }
3963
3964    #[tokio::test]
3965    async fn stale_in_review_recovery_leaves_decided_task_waiting_on_merge() {
3966        assert_eq!(
3967            orphaned_review_status_after_recovery(true).await,
3968            TaskStatus::InReview
3969        );
3970    }
3971
3972    #[tokio::test]
3973    async fn recover_stuck_tasks_resets_review_agent_failures() {
3974        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3975        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3976        let task_manager = Arc::new(TaskManager::with_store(
3977            Arc::clone(&backend),
3978            Arc::clone(&store),
3979            "owner/repo".to_string(),
3980        ));
3981        let tmux = Arc::new(TmuxManager::new());
3982
3983        let id = store
3984            .upsert_external(&crate::store::UpsertExternal {
3985                repo: "owner/repo",
3986                ext_id: "104",
3987                title: "Orphaned review with failures",
3988                body: "",
3989                author: "",
3990                url: "",
3991                labels: &[],
3992                origin: "github",
3993            })
3994            .await
3995            .unwrap();
3996        store.update_status(id, TaskStatus::InReview).await.unwrap();
3997        store
3998            .set_fields(id, &[("review_agent_failures", serde_json::json!(2))])
3999            .await
4000            .unwrap();
4001        set_in_review_entered_at_past(&store, id).await;
4002
4003        tick_recover_stuck_tasks(
4004            &backend,
4005            &tmux,
4006            "owner/repo",
4007            &task_manager,
4008            &EngineConfig::default(),
4009            &store,
4010            &HashMap::new(),
4011            &Arc::new(DashMap::new()),
4012        )
4013        .await
4014        .unwrap();
4015
4016        let task = store.get(id).await.unwrap();
4017        assert_eq!(task.status, TaskStatus::NeedsReview);
4018        assert_eq!(
4019            task.review_agent_failures, 0,
4020            "a dead review session is an infrastructure event, not an agent failure"
4021        );
4022    }
4023
4024    #[tokio::test]
4025    async fn recover_stuck_tasks_reclaims_hung_in_review_session() {
4026        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4027        let mock = MockBackend::new();
4028        let status_updates = Arc::clone(&mock.status_updates);
4029        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4030        let task_manager = Arc::new(TaskManager::with_store(
4031            Arc::clone(&backend),
4032            Arc::clone(&store),
4033            "owner/repo".to_string(),
4034        ));
4035        let tmux = Arc::new(TmuxManager::new());
4036        let config = EngineConfig {
4037            no_session_stuck_timeout: 600,
4038            stuck_timeout: 1800,
4039            ..EngineConfig::default()
4040        };
4041
4042        let id = store
4043            .upsert_external(&crate::store::UpsertExternal {
4044                repo: "owner/repo",
4045                ext_id: "103",
4046                title: "Hung review session",
4047                body: "",
4048                author: "",
4049                url: "",
4050                labels: &[],
4051                origin: "github",
4052            })
4053            .await
4054            .unwrap();
4055        store.update_status(id, TaskStatus::InReview).await.unwrap();
4056        set_in_review_entered_at_past(&store, id).await;
4057        store
4058            .set_fields(id, &[("pr_number", serde_json::json!(43))])
4059            .await
4060            .unwrap();
4061
4062        let session_name = tmux.session_name("owner/repo", "103-review");
4063        let mut session_map = HashMap::new();
4064        session_map.insert(session_name, true);
4065
4066        tick_recover_stuck_tasks(
4067            &backend,
4068            &tmux,
4069            "owner/repo",
4070            &task_manager,
4071            &config,
4072            &store,
4073            &session_map,
4074            &Arc::new(DashMap::new()),
4075        )
4076        .await
4077        .unwrap();
4078
4079        let updates = status_updates.lock().unwrap();
4080        assert_eq!(
4081            updates.len(),
4082            1,
4083            "a live hung review session must be reclaimed once the in_review entry ages past stuck_timeout"
4084        );
4085        assert_eq!(updates[0].0, "103");
4086        assert_eq!(updates[0].1, Status::NeedsReview);
4087    }
4088
4089    #[tokio::test]
4090    async fn untracked_repo_recovery_resets_stale_in_progress_to_new() {
4091        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4092        let tmux = Arc::new(TmuxManager::new());
4093        let config = EngineConfig {
4094            no_session_stuck_timeout: 600,
4095            stuck_timeout: 1800,
4096            ..EngineConfig::default()
4097        };
4098
4099        // Cross-repo task should still be recovered even when current tick repo is owner/repo.
4100        let id = store
4101            .upsert_external(&crate::store::UpsertExternal {
4102                repo: "owner/other",
4103                ext_id: "777",
4104                title: "Cross-repo stuck task",
4105                body: "",
4106                author: "",
4107                url: "",
4108                labels: &[],
4109                origin: "github",
4110            })
4111            .await
4112            .unwrap();
4113        store
4114            .update_status(id, TaskStatus::InProgress)
4115            .await
4116            .unwrap();
4117        set_task_updated_at_past(&store, id).await;
4118
4119        let active = HashSet::from(["owner/repo".to_string()]);
4120        recover_untracked_repo_stuck_tasks(
4121            &tmux,
4122            &config,
4123            &store,
4124            &HashMap::new(),
4125            &Arc::new(DashMap::new()),
4126            &active,
4127        )
4128        .await;
4129
4130        let task = store.get(id).await.unwrap();
4131        assert_eq!(
4132            task.status,
4133            TaskStatus::New,
4134            "cross-repo stale in_progress task should be reset to New"
4135        );
4136        assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
4137        assert!(task.agent.is_none(), "agent should be cleared");
4138        assert!(task.model.is_none(), "model should be cleared");
4139    }
4140
4141    #[tokio::test]
4142    async fn untracked_repo_recovery_leaves_active_repos_to_their_own_pass() {
4143        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4144        let tmux = Arc::new(TmuxManager::new());
4145        let config = EngineConfig {
4146            no_session_stuck_timeout: 600,
4147            stuck_timeout: 1800,
4148            ..EngineConfig::default()
4149        };
4150        let id = store
4151            .upsert_external(&crate::store::UpsertExternal {
4152                repo: "owner/other",
4153                ext_id: "778",
4154                title: "Stuck task in an active repo",
4155                body: "",
4156                author: "",
4157                url: "",
4158                labels: &[],
4159                origin: "github",
4160            })
4161            .await
4162            .unwrap();
4163        store
4164            .update_status(id, TaskStatus::InProgress)
4165            .await
4166            .unwrap();
4167        set_task_updated_at_past(&store, id).await;
4168
4169        // Per-project passes run concurrently, so only the task's own pass may reclaim it
4170        let active = HashSet::from(["owner/repo".to_string(), "owner/other".to_string()]);
4171        recover_untracked_repo_stuck_tasks(
4172            &tmux,
4173            &config,
4174            &store,
4175            &HashMap::new(),
4176            &Arc::new(DashMap::new()),
4177            &active,
4178        )
4179        .await;
4180
4181        assert_eq!(store.get(id).await.unwrap().status, TaskStatus::InProgress);
4182    }
4183
4184    #[tokio::test]
4185    async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
4186        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4187        let mock = MockBackend::new();
4188        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4189        let task_manager = Arc::new(TaskManager::with_store(
4190            Arc::clone(&backend),
4191            Arc::clone(&store),
4192            "owner/repo".to_string(),
4193        ));
4194        let tmux = Arc::new(TmuxManager::new());
4195        let config = EngineConfig {
4196            no_session_stuck_timeout: 600,
4197            stuck_timeout: 1800,
4198            ..EngineConfig::default()
4199        };
4200
4201        let id = store
4202            .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
4203            .await
4204            .unwrap();
4205        store
4206            .set_fields(
4207                id,
4208                &[
4209                    ("agent", serde_json::json!("claude")),
4210                    ("model", serde_json::json!("sonnet")),
4211                    ("route_attempts", serde_json::json!(1)),
4212                ],
4213            )
4214            .await
4215            .unwrap();
4216        store
4217            .update_status(id, TaskStatus::InProgress)
4218            .await
4219            .unwrap();
4220        set_task_updated_at_past(&store, id).await;
4221
4222        // Start an agent run that will be orphaned by recovery
4223        store
4224            .start_run(&crate::store::StartRun {
4225                task_id: id,
4226                attempt: 1,
4227                run_type: "agent",
4228                agent: "claude",
4229                model: "sonnet",
4230                command: "claude --model sonnet",
4231                prompt: "fix it",
4232            })
4233            .await
4234            .unwrap();
4235
4236        tick_recover_stuck_tasks(
4237            &backend,
4238            &tmux,
4239            "owner/repo",
4240            &task_manager,
4241            &config,
4242            &store,
4243            &HashMap::new(),
4244            &Arc::new(DashMap::new()),
4245        )
4246        .await
4247        .unwrap();
4248
4249        let task = store.get(id).await.unwrap();
4250        assert_eq!(task.status, TaskStatus::New);
4251
4252        let runs = store.get_runs(id).await.unwrap();
4253        assert_eq!(runs.len(), 1);
4254        assert_eq!(runs[0].outcome, "aborted");
4255        assert!(runs[0].completed_at.is_some());
4256        assert_eq!(
4257            runs[0].error,
4258            "stuck-task recovery: internal no session found"
4259        );
4260    }
4261
4262    /// Regression test for #3518: a completed agent run must not be discarded by the
4263    /// stuck-task reclaim just because its tmux session has already exited.
4264    ///
4265    /// In production, sessions are created without `remain-on-exit`, so once the agent
4266    /// process exits tmux removes the session immediately — there is no observable
4267    /// "dead pane" state for `tick_check_session_completions` to catch, and the runner's
4268    /// own `store_touch_updated_at` only fires after its own poll notices the exit (up to
4269    /// its 5s poll interval) plus post-processing. For a task whose total runtime already
4270    /// exceeds `no_session_stuck_timeout`, the very next tick that observes the (now
4271    /// absent) session would otherwise reclaim it instantly — discarding a successful run.
4272    ///
4273    /// The fix: `tick_recover_stuck_tasks` skips the no-session reclaim while the task's
4274    /// dispatch guard (the in-memory `dispatching` map keyed by "{repo}/{task_id}",
4275    /// populated for the entire duration of `runner.run_with_context`) is still held —
4276    /// i.e. while this process is still actively running the task, regardless of tmux
4277    /// session state.
4278    #[tokio::test]
4279    async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
4280        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4281        let mock = MockBackend::new();
4282        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4283        let task_manager = Arc::new(TaskManager::with_store(
4284            Arc::clone(&backend),
4285            Arc::clone(&store),
4286            "owner/repo".to_string(),
4287        ));
4288        let tmux = Arc::new(TmuxManager::new());
4289        let config = EngineConfig {
4290            no_session_stuck_timeout: 600,
4291            stuck_timeout: 1800,
4292            ..EngineConfig::default()
4293        };
4294
4295        let id = store
4296            .create_internal(
4297                "owner/repo",
4298                "Long-running daily job",
4299                "",
4300                "cron",
4301                "1",
4302                None,
4303            )
4304            .await
4305            .unwrap();
4306        store
4307            .set_fields(
4308                id,
4309                &[
4310                    ("agent", serde_json::json!("claude")),
4311                    ("model", serde_json::json!("sonnet")),
4312                    ("route_attempts", serde_json::json!(1)),
4313                ],
4314            )
4315            .await
4316            .unwrap();
4317        store
4318            .update_status(id, TaskStatus::InProgress)
4319            .await
4320            .unwrap();
4321        // Simulate a task that has already run well past no_session_stuck_timeout,
4322        // as a legitimately long agent run would.
4323        set_task_updated_at_past(&store, id).await;
4324
4325        // Simulate the runner still actively processing this task: its dispatch guard
4326        // is held even though the tmux session (empty session_map below) is already gone.
4327        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4328        dispatching.insert(
4329            format!("owner/repo/internal:{id}"),
4330            format!("internal:{id}"),
4331        );
4332
4333        tick_recover_stuck_tasks(
4334            &backend,
4335            &tmux,
4336            "owner/repo",
4337            &task_manager,
4338            &config,
4339            &store,
4340            &HashMap::new(),
4341            &dispatching,
4342        )
4343        .await
4344        .unwrap();
4345
4346        let task = store.get(id).await.unwrap();
4347        assert_eq!(
4348            task.status,
4349            TaskStatus::InProgress,
4350            "task should not be reclaimed while its dispatch guard is still held"
4351        );
4352
4353        // No run should have been finalized/aborted either.
4354        let runs = store.get_runs(id).await.unwrap();
4355        assert!(runs.is_empty());
4356
4357        // Once the guard is released (runner finished), the very same conditions should
4358        // reclaim the task as before — the fix must not disable the safety net entirely.
4359        dispatching.clear();
4360        tick_recover_stuck_tasks(
4361            &backend,
4362            &tmux,
4363            "owner/repo",
4364            &task_manager,
4365            &config,
4366            &store,
4367            &HashMap::new(),
4368            &dispatching,
4369        )
4370        .await
4371        .unwrap();
4372        let task = store.get(id).await.unwrap();
4373        assert_eq!(
4374            task.status,
4375            TaskStatus::New,
4376            "task should still be reclaimed once the dispatch guard is released"
4377        );
4378    }
4379
4380    /// Concurrency regression test for issue #3526: the reclaim race recurred twice
4381    /// (internal:156563 in #3523, internal:156673 in #3526) *after* the #3519/#3525
4382    /// fix landed, in production under the real multi-threaded tokio runtime. The
4383    /// existing `recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held` test
4384    /// above only exercises sequential `.await` calls on a single-threaded test
4385    /// runtime (`#[tokio::test]` defaults to `flavor = "current_thread"`), which can
4386    /// never observe genuine cross-thread races in the shared `dispatching` DashMap.
4387    ///
4388    /// This test claims the guard exactly the way `dispatch_tasks_for_repo` does
4389    /// (atomic `DashMap::entry` claim, `DispatchGuard` RAII release), hands it to a
4390    /// spawned task running on a *different* worker thread that holds it until
4391    /// explicitly told to release, and hammers the real `tick_recover_stuck_tasks`
4392    /// from the main test task hundreds of times while the guard is deterministically
4393    /// still held. If there were a visibility gap between `DashMap::entry().insert()`
4394    /// on one thread and `DashMap::contains_key()` on another, this test would catch it.
4395    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
4396    async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
4397        const HAMMER_ITERATIONS: u32 = 300;
4398
4399        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4400        let mock = MockBackend::new();
4401        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4402        let task_manager = Arc::new(TaskManager::with_store(
4403            Arc::clone(&backend),
4404            Arc::clone(&store),
4405            "owner/repo".to_string(),
4406        ));
4407        let tmux = Arc::new(TmuxManager::new());
4408        let config = EngineConfig {
4409            no_session_stuck_timeout: 600,
4410            stuck_timeout: 1800,
4411            ..EngineConfig::default()
4412        };
4413
4414        let id = store
4415            .create_internal(
4416                "owner/repo",
4417                "Long-running daily job",
4418                "",
4419                "cron",
4420                "1",
4421                None,
4422            )
4423            .await
4424            .unwrap();
4425        store
4426            .set_fields(
4427                id,
4428                &[
4429                    ("agent", serde_json::json!("claude")),
4430                    ("model", serde_json::json!("sonnet")),
4431                    ("route_attempts", serde_json::json!(1)),
4432                ],
4433            )
4434            .await
4435            .unwrap();
4436        store
4437            .update_status(id, TaskStatus::InProgress)
4438            .await
4439            .unwrap();
4440        set_task_updated_at_past(&store, id).await;
4441
4442        let dispatch_key = format!("owner/repo/internal:{id}");
4443        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4444
4445        // Claim the guard synchronously, on this task, exactly like
4446        // dispatch_tasks_for_repo does: the atomic entry-claim always happens before
4447        // tokio::spawn hands the guard to the background task. (Claiming inside the
4448        // spawned task itself would race the hammer loop below against tokio's own
4449        // scheduling of that task onto a worker thread — a test artifact, not the
4450        // real production ordering, which guarantees claim-before-spawn.)
4451        let claim_task_id = format!("internal:{id}");
4452        {
4453            match dispatching.entry(dispatch_key.clone()) {
4454                Entry::Vacant(slot) => {
4455                    slot.insert(claim_task_id);
4456                }
4457                Entry::Occupied(_) => panic!("dispatch key already claimed"),
4458            }
4459        }
4460        let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
4461
4462        // Hold the guard on a spawned task — a different worker thread under the
4463        // multi-thread runtime — mirroring the real runner holding it across genuine
4464        // async work (worktree setup, agent session, response handling) inside
4465        // `tokio::spawn(... run_with_context ...)`. It only drops the guard once
4466        // explicitly told to via `release_rx`, so its lifetime is fully controlled
4467        // by this test rather than raced against a timer — the release-signal
4468        // ordering was itself a source of false positives in an earlier version of
4469        // this test (a `sleep`-then-drop holder can legitimately release the guard
4470        // in the gap between the hammer loop's last check and its next iteration,
4471        // which is a correct outcome, not a bug, but made the assertion flaky).
4472        let (release_tx, release_rx) = oneshot::channel::<()>();
4473        let holder = tokio::spawn(async move {
4474            let _ = release_rx.await;
4475            drop(guard);
4476        });
4477
4478        // Hammer the real reclaim path concurrently from the main task while the
4479        // guard is deterministically still held (the holder task is parked on
4480        // `release_rx` and cannot drop the guard until we signal it below). Each
4481        // iteration's `tick_recover_stuck_tasks` call may itself run on any worker
4482        // thread the tokio scheduler picks, independent of the thread that claimed
4483        // the guard — this is what actually exercises cross-thread DashMap
4484        // visibility, not wall-clock racing against a timer.
4485        for i in 0..HAMMER_ITERATIONS {
4486            tick_recover_stuck_tasks(
4487                &backend,
4488                &tmux,
4489                "owner/repo",
4490                &task_manager,
4491                &config,
4492                &store,
4493                &HashMap::new(),
4494                &dispatching,
4495            )
4496            .await
4497            .unwrap();
4498
4499            let task = store.get(id).await.unwrap();
4500            assert_eq!(
4501                task.status,
4502                TaskStatus::InProgress,
4503                "task must never be reclaimed while the dispatch guard is deterministically \
4504                 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
4505                 iteration={i}, contains_key_now={}, map_len_now={})",
4506                dispatching.contains_key(&dispatch_key),
4507                dispatching.len(),
4508            );
4509            task::yield_now().await;
4510        }
4511
4512        // Now release the guard and confirm the holder task actually completed.
4513        let _ = release_tx.send(());
4514        holder.await.unwrap();
4515
4516        // Guard released — the safety net must still reclaim as before.
4517        tick_recover_stuck_tasks(
4518            &backend,
4519            &tmux,
4520            "owner/repo",
4521            &task_manager,
4522            &config,
4523            &store,
4524            &HashMap::new(),
4525            &dispatching,
4526        )
4527        .await
4528        .unwrap();
4529        let task = store.get(id).await.unwrap();
4530        assert_eq!(
4531            task.status,
4532            TaskStatus::New,
4533            "task should be reclaimed once the dispatch guard is released"
4534        );
4535    }
4536
4537    #[tokio::test]
4538    async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
4539        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4540        let mock = MockBackend::new();
4541        let status_updates = Arc::clone(&mock.status_updates);
4542        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4543        let task_manager = Arc::new(TaskManager::with_store(
4544            Arc::clone(&backend),
4545            Arc::clone(&store),
4546            "owner/repo".to_string(),
4547        ));
4548        let tmux = Arc::new(TmuxManager::new());
4549        let config = EngineConfig {
4550            no_session_stuck_timeout: 600,
4551            stuck_timeout: 1800,
4552            ..EngineConfig::default()
4553        };
4554
4555        let id = store
4556            .upsert_external(&crate::store::UpsertExternal {
4557                repo: "owner/repo",
4558                ext_id: "101",
4559                title: "Waiting For Human Review",
4560                body: "",
4561                author: "",
4562                url: "",
4563                labels: &[],
4564                origin: "github",
4565            })
4566            .await
4567            .unwrap();
4568        store.update_status(id, TaskStatus::InReview).await.unwrap();
4569        set_task_updated_at_past(&store, id).await;
4570
4571        tick_recover_stuck_tasks(
4572            &backend,
4573            &tmux,
4574            "owner/repo",
4575            &task_manager,
4576            &config,
4577            &store,
4578            &HashMap::new(),
4579            &Arc::new(DashMap::new()),
4580        )
4581        .await
4582        .unwrap();
4583
4584        let updates = status_updates.lock().unwrap();
4585        assert!(
4586            updates.is_empty(),
4587            "InReview task waiting for human review should not be reset"
4588        );
4589    }
4590
4591    #[tokio::test]
4592    async fn unblock_parents_no_blocked_tasks() {
4593        let mock = MockBackend::new(); // no blocked tasks
4594        let status_updates = Arc::clone(&mock.status_updates);
4595        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4596        let task_manager = make_task_manager(Arc::clone(&backend));
4597        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4598
4599        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4600            .await
4601            .unwrap();
4602
4603        assert!(status_updates.lock().unwrap().is_empty());
4604    }
4605
4606    #[tokio::test]
4607    async fn unblock_parents_handles_multiple_blocked_tasks() {
4608        let mut mock = MockBackend::new();
4609
4610        // Task 40: all children done → should unblock
4611        let t40 = make_task("40", &["status:blocked"]);
4612        mock.blocked_tasks.push(t40);
4613        mock.sub_issues
4614            .insert("40".to_string(), vec![ExternalId("41".to_string())]);
4615        mock.tasks_by_id
4616            .insert("41".to_string(), make_task("41", &["status:done"]));
4617
4618        // Task 50: child not done → should stay blocked
4619        let t50 = make_task("50", &["status:blocked"]);
4620        mock.blocked_tasks.push(t50);
4621        mock.sub_issues
4622            .insert("50".to_string(), vec![ExternalId("51".to_string())]);
4623        mock.tasks_by_id
4624            .insert("51".to_string(), make_task("51", &["status:in_progress"]));
4625
4626        let status_updates = Arc::clone(&mock.status_updates);
4627        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4628        let task_manager = make_task_manager(Arc::clone(&backend));
4629        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4630
4631        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4632            .await
4633            .unwrap();
4634
4635        let updates = status_updates.lock().unwrap();
4636        assert_eq!(updates.len(), 1, "only one parent should be unblocked");
4637        assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
4638        assert_eq!(updates[0].1, Status::New);
4639    }
4640
4641    /// `tick_all` reads leadership and the global sweeps from the first project's store, so
4642    /// an empty project list must return before any of that, and touch nothing.
4643    #[tokio::test]
4644    async fn tick_all_is_a_no_op_without_projects() {
4645        let transport = Arc::new(Transport::new());
4646        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4647        let tmux = Arc::new(TmuxManager::new());
4648        let semaphore = Arc::new(Semaphore::new(1));
4649        let router = Arc::new(RwLock::new(cooled_review_router_for_test()));
4650        let (weight_tx, mut weight_rx) = mpsc::channel(8);
4651        let dispatching = Arc::new(DashMap::new());
4652        let auto_merge_in_flight = Arc::new(DashSet::new());
4653
4654        tick_all(
4655            &[],
4656            &tmux,
4657            &capture,
4658            &semaphore,
4659            &EngineConfig::default(),
4660            &router,
4661            &weight_tx,
4662            &dispatching,
4663            Some(&transport),
4664            &HashSet::new(),
4665            &auto_merge_in_flight,
4666        )
4667        .await;
4668
4669        assert!(dispatching.is_empty(), "no project, so no dispatch claimed");
4670        assert!(
4671            weight_rx.try_recv().is_err(),
4672            "no project, so no weight signal"
4673        );
4674    }
4675
4676    /// The store-wide sweeps must run once per tick, not once per project. The NeedsReview
4677    /// refire sweep reads `needs_review_refires` and then increments it, so when it ran
4678    /// inside the per-project join every project saw the same count and spent one of the
4679    /// five refire attempts — four projects burned four attempts in a single tick (#3864).
4680    #[serial_test::serial(cooldown_state)]
4681    #[tokio::test]
4682    async fn tick_all_runs_the_global_needs_review_sweep_once_for_two_projects() {
4683        cooldown::reset_global_state().await;
4684
4685        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4686        let id = store
4687            .upsert_external(&UpsertExternal {
4688                repo: "owner/one",
4689                ext_id: "7",
4690                title: "Stale needs_review task",
4691                body: "",
4692                author: "user",
4693                url: "https://github.com/owner/one/issues/7",
4694                labels: &[],
4695                origin: "github",
4696            })
4697            .await
4698            .unwrap();
4699        store
4700            .update_status(id, TaskStatus::NeedsReview)
4701            .await
4702            .unwrap();
4703        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4704            .bind(id)
4705            .execute(store.pool())
4706            .await
4707            .unwrap();
4708
4709        let engines: Vec<ProjectEngine> = ["owner/one", "owner/two"]
4710            .iter()
4711            .map(|repo| {
4712                let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4713                ProjectEngine {
4714                    repo: repo.to_string(),
4715                    project_dir: PathBuf::from("/nonexistent"),
4716                    backend: Arc::clone(&backend),
4717                    task_manager: Arc::new(TaskManager::with_store(
4718                        backend,
4719                        Arc::clone(&store),
4720                        repo.to_string(),
4721                    )),
4722                    runner: Arc::new(TaskRunner::new(repo.to_string())),
4723                    store: Arc::clone(&store),
4724                }
4725            })
4726            .collect();
4727
4728        // A healthy review agent, so the sweep does not skip the pass
4729        let mut config = RouterConfig::default();
4730        config
4731            .model_map
4732            .entry("review".to_string())
4733            .or_default()
4734            .insert("claude".to_string(), vec!["sonnet".to_string()]);
4735        let router = Arc::new(RwLock::new(Router::new_for_test(
4736            config,
4737            vec!["claude".to_string()],
4738        )));
4739
4740        let transport = Arc::new(Transport::new());
4741        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4742        let (weight_tx, _weight_rx) = mpsc::channel(8);
4743
4744        tick_all(
4745            &engines,
4746            &Arc::new(TmuxManager::new()),
4747            &capture,
4748            &Arc::new(Semaphore::new(1)),
4749            &EngineConfig::default(),
4750            &router,
4751            &weight_tx,
4752            &Arc::new(DashMap::new()),
4753            Some(&transport),
4754            &engines.iter().map(|e| e.repo.clone()).collect(),
4755            &Arc::new(DashSet::new()),
4756        )
4757        .await;
4758
4759        assert_eq!(
4760            store.get(id).await.unwrap().needs_review_refires,
4761            1,
4762            "two projects must share one pass of the global NeedsReview sweep"
4763        );
4764    }
4765
4766    /// `tick_all` must not reap tmux sessions it did not start. Startup cleanup used to
4767    /// live here, and nextest's per-test process made the once-flag kill live agents (#3889)
4768    #[tokio::test]
4769    async fn tick_all_does_not_kill_sessions_it_did_not_create() {
4770        let tmux = Arc::new(TmuxManager::new());
4771        let Ok(session_name) = tmux
4772            .create_session("other/foreign", "3889", "/tmp", "sleep 30", &[])
4773            .await
4774        else {
4775            eprintln!("Skipping test: tmux not available or failed to create test session");
4776            return;
4777        };
4778        let _guard = SessionGuard(session_name.clone());
4779
4780        // Cross a unix-second boundary so a reintroduced startup_cleanup would treat
4781        // this session as stale (created_at < now)
4782        time::sleep(Duration::from_millis(1100)).await;
4783
4784        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4785        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4786        let engines = vec![ProjectEngine {
4787            repo: "owner/one".to_string(),
4788            project_dir: PathBuf::from("/nonexistent"),
4789            backend: Arc::clone(&backend),
4790            task_manager: Arc::new(TaskManager::with_store(
4791                backend,
4792                Arc::clone(&store),
4793                "owner/one".to_string(),
4794            )),
4795            runner: Arc::new(TaskRunner::new("owner/one".to_string())),
4796            store,
4797        }];
4798
4799        let transport = Arc::new(Transport::new());
4800        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4801        let (weight_tx, _weight_rx) = mpsc::channel(8);
4802        let router = Arc::new(RwLock::new(cooled_review_router_for_test()));
4803
4804        tick_all(
4805            &engines,
4806            &tmux,
4807            &capture,
4808            &Arc::new(Semaphore::new(1)),
4809            &EngineConfig::default(),
4810            &router,
4811            &weight_tx,
4812            &Arc::new(DashMap::new()),
4813            Some(&transport),
4814            &engines.iter().map(|e| e.repo.clone()).collect(),
4815            &Arc::new(DashSet::new()),
4816        )
4817        .await;
4818
4819        assert!(
4820            tmux.session_exists(&session_name).await,
4821            "tick_all must not kill a tmux session it did not start"
4822        );
4823    }
4824
4825    #[tokio::test]
4826    async fn detect_silent_agents_skips_backend_for_internal_tasks() {
4827        let transport = Arc::new(Transport::new());
4828        let capture = Arc::new(CaptureService::new(transport));
4829        let mock = Arc::new(MockBackend::new());
4830        let backend: Arc<dyn ExternalBackend> = Arc::<MockBackend>::clone(&mock);
4831        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4832        let task_manager = Arc::new(TaskManager::with_store(
4833            Arc::clone(&backend),
4834            Arc::clone(&store),
4835            "owner/repo".to_string(),
4836        ));
4837        let tmux = Arc::new(TmuxManager::new());
4838        let config = EngineConfig {
4839            silence_grace_period: 0,
4840            ..EngineConfig::default()
4841        };
4842
4843        let internal_id = store
4844            .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
4845            .await
4846            .unwrap();
4847        store
4848            .update_status(internal_id, TaskStatus::InProgress)
4849            .await
4850            .unwrap();
4851
4852        let task_id = format!("internal:{internal_id}");
4853        capture
4854            .register_session("owner/repo", &task_id, "orch-test-internal")
4855            .await;
4856        time::sleep(Duration::from_millis(1100)).await;
4857
4858        tick_detect_silent_agents(
4859            &tmux,
4860            "owner/repo",
4861            &capture,
4862            &backend,
4863            &task_manager,
4864            &config,
4865            &store,
4866        )
4867        .await
4868        .unwrap();
4869
4870        assert!(
4871            mock.removed_labels.lock().unwrap().is_empty(),
4872            "internal tasks should skip backend label removal"
4873        );
4874        assert!(
4875            mock.posted_comments.lock().unwrap().is_empty(),
4876            "internal tasks should skip backend comments"
4877        );
4878
4879        let task = store.get(internal_id).await.unwrap();
4880        // Silence detection now uses failover: if a fallback agent is found → Routed,
4881        // otherwise → New (reset for re-routing when cooldowns expire). Either is
4882        // acceptable; the key invariant is that the task is no longer InProgress.
4883        assert!(
4884            matches!(task.status, TaskStatus::Routed | TaskStatus::New),
4885            "internal task should be routed to fallback or reset to new, got {:?}",
4886            task.status
4887        );
4888    }
4889
4890    #[tokio::test]
4891    async fn detect_silent_agents_finalizes_open_task_runs_rows() {
4892        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4893        let mock = MockBackend::new();
4894        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4895        let task_manager = Arc::new(TaskManager::with_store(
4896            Arc::clone(&backend),
4897            Arc::clone(&store),
4898            "owner/repo".to_string(),
4899        ));
4900        let transport = Arc::new(Transport::new());
4901        let capture = Arc::new(CaptureService::new(transport));
4902        let tmux = Arc::new(TmuxManager::new());
4903        let config = EngineConfig {
4904            silence_grace_period: 60,
4905            silence_cooldown: 300,
4906            ..EngineConfig::default()
4907        };
4908
4909        let internal_id = store
4910            .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4911            .await
4912            .unwrap();
4913        store
4914            .set_fields(
4915                internal_id,
4916                &[
4917                    ("agent", serde_json::json!("claude")),
4918                    ("model", serde_json::json!("sonnet")),
4919                ],
4920            )
4921            .await
4922            .unwrap();
4923        store
4924            .update_status(internal_id, TaskStatus::InProgress)
4925            .await
4926            .unwrap();
4927
4928        let task_id = format!("internal:{internal_id}");
4929        capture
4930            .register_session("owner/repo", &task_id, "orch-test-silent-run")
4931            .await;
4932        capture
4933            .set_buffer_state_for_test(
4934                "owner/repo",
4935                &task_id,
4936                false,
4937                false,
4938                chrono::Utc::now() - chrono::Duration::seconds(120),
4939            )
4940            .await;
4941
4942        // Start an agent run that will be orphaned by silence detection
4943        store
4944            .start_run(&crate::store::StartRun {
4945                task_id: internal_id,
4946                attempt: 1,
4947                run_type: "agent",
4948                agent: "claude",
4949                model: "sonnet",
4950                command: "claude --model sonnet",
4951                prompt: "fix it",
4952            })
4953            .await
4954            .unwrap();
4955
4956        tick_detect_silent_agents(
4957            &tmux,
4958            "owner/repo",
4959            &capture,
4960            &backend,
4961            &task_manager,
4962            &config,
4963            &store,
4964        )
4965        .await
4966        .unwrap();
4967
4968        let runs = store.get_runs(internal_id).await.unwrap();
4969        assert_eq!(runs.len(), 1);
4970        assert_eq!(runs[0].outcome, "aborted");
4971        assert!(runs[0].completed_at.is_some());
4972        assert_eq!(runs[0].error, "silence detection: session killed");
4973    }
4974
4975    /// Regression test for the race between Phase 1 (tick_check_session_completions)
4976    /// and Phase 2 (tick_recover_stuck_tasks).
4977    ///
4978    /// Before the fix, Phase 1 killed the tmux session without touching updated_at.
4979    /// Phase 2 could then see status=in_progress, has_session=false, and an old
4980    /// updated_at, and would reset the task to New — even though the runner's
4981    /// wait_for_completion() poll had not yet returned.
4982    ///
4983    /// The fix: Phase 1 touches updated_at as soon as it detects the session is done.
4984    /// This test verifies that after the touch, Phase 2 does NOT reclaim the task.
4985    #[tokio::test]
4986    async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4987        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4988        let mock = MockBackend::new();
4989        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4990        let task_manager = Arc::new(TaskManager::with_store(
4991            Arc::clone(&backend),
4992            Arc::clone(&store),
4993            "owner/repo".to_string(),
4994        ));
4995        let tmux = Arc::new(TmuxManager::new());
4996        let config = EngineConfig {
4997            no_session_stuck_timeout: 600,
4998            stuck_timeout: 1800,
4999            ..EngineConfig::default()
5000        };
5001
5002        // Create an in-progress internal task with old updated_at.
5003        // Without the fix this task would be reclaimed by Phase 2 (age >> threshold,
5004        // no session running in the test environment).
5005        let id = store
5006            .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
5007            .await
5008            .unwrap();
5009        store
5010            .update_status(id, TaskStatus::InProgress)
5011            .await
5012            .unwrap();
5013        set_task_updated_at_past(&store, id).await;
5014
5015        // Simulate Phase 1: touch updated_at when the session is detected as done.
5016        // In production this happens inside tick_check_session_completions.
5017        let task_id = format!("internal:{id}");
5018        store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
5019
5020        // Phase 2 runs — updated_at is fresh so the task must NOT be reclaimed.
5021        tick_recover_stuck_tasks(
5022            &backend,
5023            &tmux,
5024            "owner/repo",
5025            &task_manager,
5026            &config,
5027            &store,
5028            &HashMap::new(),
5029            &Arc::new(DashMap::new()),
5030        )
5031        .await
5032        .unwrap();
5033
5034        let task = store.get(id).await.unwrap();
5035        assert_eq!(
5036            task.status,
5037            TaskStatus::InProgress,
5038            "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
5039        );
5040    }
5041
5042    #[tokio::test]
5043    async fn stuck_task_timing_treats_dead_session_as_missing() {
5044        let tmux = Arc::new(TmuxManager::new());
5045        let repo = "owner/repo";
5046        let task_id = "internal:33498";
5047        let session_name = tmux.session_name(repo, task_id);
5048
5049        let create_result = tmux
5050            .command()
5051            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5052            .output()
5053            .await;
5054
5055        match create_result {
5056            Ok(o) if o.status.success() => {}
5057            _ => {
5058                eprintln!("Skipping test: tmux not available or failed to create test session");
5059                return;
5060            }
5061        }
5062
5063        let _guard = SessionGuard(session_name.clone());
5064
5065        let set_option_result = tmux
5066            .command()
5067            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5068            .output()
5069            .await;
5070        if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
5071            eprintln!("Skipping test: unable to set tmux remain-on-exit option");
5072            return;
5073        }
5074
5075        let send_exit_result = tmux
5076            .command()
5077            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5078            .output()
5079            .await;
5080        if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
5081            eprintln!("Skipping test: unable to exit tmux pane");
5082            return;
5083        }
5084
5085        // Poll until pane is dead (up to 2s) — fixed sleep is unreliable under load.
5086        for _ in 0..20 {
5087            time::sleep(Duration::from_millis(100)).await;
5088            if !tmux.session_is_running(&session_name).await {
5089                break;
5090            }
5091        }
5092
5093        // Also wait for batch_session_active to reflect the dead state.
5094        // session_is_running uses `list-panes -t <session>` which may settle
5095        // before the global `list-panes -a` snapshot used by the production code.
5096        // On CI under load these two tmux commands can transiently disagree.
5097        let mut session_map = tmux.batch_session_active().await;
5098        for _ in 0..20 {
5099            if !session_map.get(&session_name).copied().unwrap_or(false) {
5100                break;
5101            }
5102            time::sleep(Duration::from_millis(100)).await;
5103            session_map = tmux.batch_session_active().await;
5104        }
5105        if session_map.get(&session_name).copied().unwrap_or(false) {
5106            eprintln!(
5107                "Skipping test: session still alive in batch_session_active after retries (CI tmux lag)"
5108            );
5109            return;
5110        }
5111
5112        let config = EngineConfig {
5113            no_session_stuck_timeout: 600,
5114            stuck_timeout: 1800,
5115            ..EngineConfig::default()
5116        };
5117        let updated_at = (chrono::Utc::now()
5118            - chrono::Duration::seconds(
5119                i64::try_from(config.no_session_stuck_timeout).expect("timeout fits in i64") + 5,
5120            ))
5121        .to_rfc3339();
5122
5123        let timing = stuck_task_timing_from_map(
5124            &tmux,
5125            repo,
5126            task_id,
5127            &updated_at,
5128            &config,
5129            "parse failure",
5130            &session_map,
5131        )
5132        .expect("dead session should be treated as missing");
5133
5134        assert!(!timing.has_session);
5135        assert_eq!(timing.threshold, config.no_session_stuck_timeout);
5136    }
5137
5138    #[tokio::test]
5139    #[serial_test::serial(suspend_state)]
5140    async fn stuck_task_timing_discounts_host_suspend_gap() {
5141        super::super::suspend::clear_for_test();
5142
5143        let tmux = Arc::new(TmuxManager::new());
5144        let repo = "owner/repo";
5145        let task_id = "internal:99001";
5146        let session_map = HashMap::new();
5147
5148        let config = EngineConfig {
5149            no_session_stuck_timeout: 1800,
5150            stuck_timeout: 1800,
5151            ..EngineConfig::default()
5152        };
5153
5154        // Task was last touched 33 minutes ago (age_mins=33 matches the reported
5155        // false-positive), but the host was suspended for 30 of those minutes —
5156        // real elapsed runtime is only 3 minutes, well under the 30-minute threshold.
5157        let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
5158        let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
5159        super::super::suspend::inject_gap_for_test(
5160            suspend_gap_detected_at,
5161            chrono::Duration::minutes(30),
5162        );
5163
5164        let timing = stuck_task_timing_from_map(
5165            &tmux,
5166            repo,
5167            task_id,
5168            &updated_at,
5169            &config,
5170            "parse failure",
5171            &session_map,
5172        );
5173
5174        assert!(
5175            timing.is_none(),
5176            "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
5177        );
5178
5179        super::super::suspend::clear_for_test();
5180    }
5181
5182    #[tokio::test]
5183    async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
5184        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5185        let mock = MockBackend::new();
5186        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5187        let task_manager = Arc::new(TaskManager::with_store(
5188            Arc::clone(&backend),
5189            Arc::clone(&store),
5190            "owner/repo".to_string(),
5191        ));
5192        let tmux = Arc::new(TmuxManager::new());
5193        let config = EngineConfig {
5194            no_session_stuck_timeout: 600,
5195            stuck_timeout: 1,
5196            ..EngineConfig::default()
5197        };
5198
5199        let id = store
5200            .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
5201            .await
5202            .unwrap();
5203        store.update_status(id, TaskStatus::Routed).await.unwrap();
5204        set_task_updated_at_past(&store, id).await;
5205
5206        let task_id = format!("internal:{id}");
5207        let session_name = tmux.session_name("owner/repo", &task_id);
5208        let create_result = tmux
5209            .command()
5210            .args([
5211                "new-session",
5212                "-d",
5213                "-s",
5214                &session_name,
5215                "-c",
5216                "/tmp",
5217                "sleep 60",
5218            ])
5219            .output()
5220            .await;
5221
5222        match create_result {
5223            Ok(o) if o.status.success() => {}
5224            _ => {
5225                eprintln!("Skipping test: tmux not available or failed to create test session");
5226                return;
5227            }
5228        }
5229
5230        let session_map = tmux.batch_session_active().await;
5231        assert_eq!(session_map.get(&session_name).copied(), Some(true));
5232
5233        tick_recover_stuck_tasks(
5234            &backend,
5235            &tmux,
5236            "owner/repo",
5237            &task_manager,
5238            &config,
5239            &store,
5240            &session_map,
5241            &Arc::new(DashMap::new()),
5242        )
5243        .await
5244        .unwrap();
5245
5246        let task = store.get(id).await.unwrap();
5247        assert_eq!(
5248            task.status,
5249            TaskStatus::New,
5250            "routed task blocked by a stale session should be reset for re-routing"
5251        );
5252        assert!(
5253            !tmux.session_exists(&session_name).await,
5254            "recovery should kill the stale tmux session so dispatch can proceed"
5255        );
5256    }
5257
5258    /// An internal complex task that entered in_progress `minutes` ago and was last written
5259    /// `updated_minutes_ago`, with a live tmux session. None when tmux is not available
5260    async fn live_complex_run(
5261        store: &TaskStore,
5262        tmux: &TmuxManager,
5263        minutes: i64,
5264        updated_minutes_ago: i64,
5265    ) -> Option<(i64, String)> {
5266        let id = store
5267            .create_internal("owner/repo", "Long complex run", "", "cron", "1", None)
5268            .await
5269            .unwrap();
5270        store
5271            .set_fields(id, &[("complexity", serde_json::json!("complex"))])
5272            .await
5273            .unwrap();
5274        store
5275            .update_status(id, TaskStatus::InProgress)
5276            .await
5277            .unwrap();
5278        let ago = |m: i64| {
5279            (chrono::Utc::now() - chrono::Duration::minutes(m))
5280                .format("%Y-%m-%dT%H:%M:%SZ")
5281                .to_string()
5282        };
5283        let entered = ago(minutes);
5284        sqlx::query(
5285            "UPDATE task_activity SET timestamp = ? WHERE task_id = ? AND to_status = 'in_progress'",
5286        )
5287        .bind(&entered)
5288        .bind(id)
5289        .execute(store.pool())
5290        .await
5291        .unwrap();
5292        sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
5293            .bind(ago(updated_minutes_ago))
5294            .bind(id)
5295            .execute(store.pool())
5296            .await
5297            .unwrap();
5298
5299        let session_name = tmux.session_name("owner/repo", &format!("internal:{id}"));
5300        let created = tmux
5301            .command()
5302            .args([
5303                "new-session",
5304                "-d",
5305                "-s",
5306                &session_name,
5307                "-c",
5308                "/tmp",
5309                "sleep 60",
5310            ])
5311            .output()
5312            .await;
5313        match created {
5314            Ok(o) if o.status.success() => Some((id, session_name)),
5315            _ => None,
5316        }
5317    }
5318
5319    fn long_run_config() -> EngineConfig {
5320        EngineConfig {
5321            stuck_timeout: 1800,
5322            run_timeouts: RunTimeouts {
5323                base: 1800,
5324                medium: 2700,
5325                complex: 3600,
5326            },
5327            ..EngineConfig::default()
5328        }
5329    }
5330
5331    #[test]
5332    fn tick_router_loads_the_config_once_for_many_live_runs() {
5333        let cell = OnceLock::new();
5334        let loads = Cell::new(0);
5335        for _ in 0..4 {
5336            tick_router(&cell, || {
5337                loads.set(loads.get() + 1);
5338                RouterConfig::default()
5339            });
5340        }
5341        assert_eq!(loads.get(), 1);
5342    }
5343
5344    #[tokio::test]
5345    async fn run_tier_follows_the_tier_of_the_stored_model() {
5346        let mut router = RouterConfig::default();
5347        router.model_map.clear();
5348        for (tier, model) in [("simple", "rt-s"), ("complex", "rt-c")] {
5349            router
5350                .model_map
5351                .entry(tier.to_string())
5352                .or_default()
5353                .insert("rtagent".to_string(), vec![model.to_string()]);
5354        }
5355        let store = TaskStore::open_memory().await.unwrap();
5356        let id = store
5357            .create_internal(
5358                "owner/repo",
5359                "Simple on a complex model",
5360                "",
5361                "cron",
5362                "1",
5363                None,
5364            )
5365            .await
5366            .unwrap();
5367        store
5368            .set_fields(
5369                id,
5370                &[
5371                    ("complexity", serde_json::json!("simple")),
5372                    ("agent", serde_json::json!("rtagent")),
5373                    ("model", serde_json::json!("rt-c")),
5374                ],
5375            )
5376            .await
5377            .unwrap();
5378        let task = store.get(id).await.unwrap();
5379        assert_eq!(run_tier(&router, &task), "complex");
5380
5381        store
5382            .set_fields(id, &[("model", serde_json::json!("rt-s"))])
5383            .await
5384            .unwrap();
5385        let task = store.get(id).await.unwrap();
5386        assert_eq!(run_tier(&router, &task), "simple");
5387    }
5388
5389    #[tokio::test]
5390    async fn recover_stuck_tasks_keeps_a_live_complex_run_inside_its_timeout() {
5391        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5392        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5393        let task_manager = Arc::new(TaskManager::with_store(
5394            Arc::clone(&backend),
5395            Arc::clone(&store),
5396            "owner/repo".to_string(),
5397        ));
5398        let tmux = Arc::new(TmuxManager::new());
5399        let Some((id, session_name)) = live_complex_run(&store, &tmux, 31, 31).await else {
5400            eprintln!("Skipping test: tmux not available or failed to create test session");
5401            return;
5402        };
5403        let session_map = tmux.batch_session_active().await;
5404
5405        tick_recover_stuck_tasks(
5406            &backend,
5407            &tmux,
5408            "owner/repo",
5409            &task_manager,
5410            &long_run_config(),
5411            &store,
5412            &session_map,
5413            &Arc::new(DashMap::new()),
5414        )
5415        .await
5416        .unwrap();
5417
5418        let alive = tmux.session_exists(&session_name).await;
5419        let _ = tmux.kill_session(&session_name).await;
5420        assert_eq!(
5421            store.get(id).await.unwrap().status,
5422            TaskStatus::InProgress,
5423            "a complex run at 31 minutes is inside its 3600s timeout"
5424        );
5425        assert!(alive, "recovery must not touch a run inside its timeout");
5426    }
5427
5428    #[tokio::test]
5429    async fn recover_stuck_tasks_kills_a_live_complex_run_past_its_timeout() {
5430        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5431        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5432        let task_manager = Arc::new(TaskManager::with_store(
5433            Arc::clone(&backend),
5434            Arc::clone(&store),
5435            "owner/repo".to_string(),
5436        ));
5437        let tmux = Arc::new(TmuxManager::new());
5438        let Some((id, session_name)) = live_complex_run(&store, &tmux, 66, 0).await else {
5439            eprintln!("Skipping test: tmux not available or failed to create test session");
5440            return;
5441        };
5442        let session_map = tmux.batch_session_active().await;
5443
5444        tick_recover_stuck_tasks(
5445            &backend,
5446            &tmux,
5447            "owner/repo",
5448            &task_manager,
5449            &long_run_config(),
5450            &store,
5451            &session_map,
5452            &Arc::new(DashMap::new()),
5453        )
5454        .await
5455        .unwrap();
5456
5457        let alive = tmux.session_exists(&session_name).await;
5458        let _ = tmux.kill_session(&session_name).await;
5459        assert_eq!(
5460            store.get(id).await.unwrap().status,
5461            TaskStatus::New,
5462            "a complex run past its timeout plus grace is stuck"
5463        );
5464        assert!(
5465            !alive,
5466            "recovery must kill the session it reports as killed"
5467        );
5468    }
5469
5470    // ── tick_dispatch_tasks: deadlock regression test (#1361) ──────────────
5471
5472    /// Regression test for #1361: tick_dispatch_tasks previously accepted
5473    /// `&Arc<RwLock<Router>>` and called `router_arc.read().await` internally.
5474    /// The main loop already held a write lock on the same RwLock, so the read
5475    /// could never be acquired — guaranteed deadlock on every tick with routed
5476    /// tasks.
5477    ///
5478    /// The fix changed the signature to accept `&Router` directly (the caller
5479    /// already has the dereferenced guard), eliminating the lock re-acquisition.
5480    ///
5481    /// This test verifies the function completes within 2 seconds when the
5482    /// caller holds the write lock — the old code would deadlock here.
5483    #[tokio::test]
5484    async fn dispatch_does_not_deadlock_under_write_lock() {
5485        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5486        let mock = MockBackend::new();
5487        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5488        let task_manager = Arc::new(TaskManager::with_store(
5489            Arc::clone(&backend),
5490            Arc::clone(&store),
5491            "owner/repo".to_string(),
5492        ));
5493        let tmux = Arc::new(TmuxManager::new());
5494        let semaphore = Arc::new(Semaphore::new(4));
5495        let (weight_tx, _weight_rx) = mpsc::channel(16);
5496        let dispatching = Arc::new(DashMap::new());
5497        let transport = Arc::new(Transport::new());
5498        let capture = Arc::new(CaptureService::new(transport));
5499
5500        let router_config = RouterConfig::default();
5501        let router = Router::new_for_test(router_config, vec!["claude".to_string()]);
5502        let router_arc = Arc::new(RwLock::new(router));
5503
5504        // Simulate what the main loop does: hold a write lock, snapshot
5505        // dispatch mode, then call tick_dispatch_tasks without passing any
5506        // lock guard into async dispatch work.
5507        let write_guard = router_arc.write().await;
5508        let dispatch_mode = dispatch_mode_from_router(&write_guard);
5509
5510        let result = time::timeout(
5511            Duration::from_secs(2),
5512            tick_dispatch_tasks(
5513                &backend,
5514                &tmux,
5515                "owner/repo",
5516                &Arc::new(TaskRunner::new("owner/repo".to_string())),
5517                &capture,
5518                &semaphore,
5519                &task_manager,
5520                &weight_tx,
5521                dispatch_mode,
5522                &dispatching,
5523                &store,
5524                &HashMap::new(),
5525            ),
5526        )
5527        .await;
5528
5529        assert!(
5530            result.is_ok(),
5531            "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
5532             on router_arc while a write lock was already held (issue #1361)"
5533        );
5534    }
5535
5536    /// Regression test for #3599: a task routed to an agent/model that entered
5537    /// cooldown *after* routing but *before* dispatch must not be dispatched —
5538    /// the stale route was decided before the failure that triggered the
5539    /// cooldown was known. Dispatch must re-check the assigned agent/model
5540    /// against the same cooldown store used at routing time and, if stale,
5541    /// leave the task in `Routed` for the next tick to re-evaluate (no PR is
5542    /// spawned, no per-task defer timer is written).
5543    #[tokio::test]
5544    async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
5545        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5546        let mock = MockBackend::new();
5547        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5548        let task_manager = Arc::new(TaskManager::with_store(
5549            Arc::clone(&backend),
5550            Arc::clone(&store),
5551            "owner/repo".to_string(),
5552        ));
5553        let tmux = Arc::new(TmuxManager::new());
5554        let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5555        let semaphore = Arc::new(Semaphore::new(4));
5556        let (weight_tx, _weight_rx) = mpsc::channel(16);
5557        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5558        let transport = Arc::new(Transport::new());
5559        let capture = Arc::new(CaptureService::new(transport));
5560
5561        let id = store
5562            .upsert_external(&UpsertExternal {
5563                repo: "owner/repo",
5564                ext_id: "999",
5565                title: "Stale routed task",
5566                body: "",
5567                author: "user",
5568                url: "https://github.com/owner/repo/issues/999",
5569                labels: &[],
5570                origin: "github",
5571            })
5572            .await
5573            .unwrap();
5574        store
5575            .store_route(&StoreRoute {
5576                id,
5577                agent: "minimax",
5578                model: Some("opus"),
5579                complexity: "medium",
5580                estimate: 0,
5581                reason: "test",
5582                profile: "{}",
5583                skills: "[]",
5584            })
5585            .await
5586            .unwrap();
5587        store.update_status(id, DbStatus::Routed).await.unwrap();
5588
5589        // Simulate a billing/rate-limit cooldown recorded for minimax:opus after
5590        // this task was routed but before it dispatched.
5591        cooldown::set_model_cooldown("minimax", "opus", 300, "rate_limit_usage_window").await;
5592
5593        let dispatch_mode = DispatchMode {
5594            is_degraded: false,
5595            healthy_agents: 1,
5596            threshold: 1,
5597        };
5598
5599        tick_dispatch_tasks(
5600            &backend,
5601            &tmux,
5602            "owner/repo",
5603            &runner,
5604            &capture,
5605            &semaphore,
5606            &task_manager,
5607            &weight_tx,
5608            dispatch_mode,
5609            &dispatching,
5610            &store,
5611            &HashMap::new(),
5612        )
5613        .await
5614        .unwrap();
5615
5616        let task = store.get(id).await.unwrap();
5617        assert_eq!(
5618            task.status,
5619            DbStatus::Routed,
5620            "task whose routed model is now cooled down must stay Routed, not be dispatched"
5621        );
5622        assert!(
5623            dispatching.is_empty(),
5624            "dispatch guard must be released when skipping a stale-cooldown task"
5625        );
5626    }
5627
5628    #[tokio::test]
5629    async fn dispatch_skips_a_task_with_an_open_dependency() {
5630        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5631        let mock = MockBackend::new();
5632        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5633        let task_manager = Arc::new(TaskManager::with_store(
5634            Arc::clone(&backend),
5635            Arc::clone(&store),
5636            "owner/repo".to_string(),
5637        ));
5638        let tmux = Arc::new(TmuxManager::new());
5639        let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5640        let semaphore = Arc::new(Semaphore::new(4));
5641        let (weight_tx, _weight_rx) = mpsc::channel(16);
5642        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5643        let transport = Arc::new(Transport::new());
5644        let capture = Arc::new(CaptureService::new(transport));
5645
5646        let id = store
5647            .upsert_external(&UpsertExternal {
5648                repo: "owner/repo",
5649                ext_id: "42",
5650                title: "Waiting routed task",
5651                body: "",
5652                author: "user",
5653                url: "https://github.com/owner/repo/issues/42",
5654                labels: &[],
5655                origin: "github",
5656            })
5657            .await
5658            .unwrap();
5659        store
5660            .replace_dependencies(
5661                id,
5662                &[NewDependency {
5663                    repo: "owner/repo".into(),
5664                    number: 7,
5665                    source: DepSource::Github,
5666                }],
5667            )
5668            .await
5669            .unwrap();
5670        store
5671            .store_route(&StoreRoute {
5672                id,
5673                agent: "claude",
5674                model: Some("haiku"),
5675                complexity: "simple",
5676                estimate: 0,
5677                reason: "test",
5678                profile: "{}",
5679                skills: "[]",
5680            })
5681            .await
5682            .unwrap();
5683        store.update_status(id, DbStatus::Routed).await.unwrap();
5684
5685        let dispatch_mode = DispatchMode {
5686            is_degraded: false,
5687            healthy_agents: 1,
5688            threshold: 1,
5689        };
5690        tick_dispatch_tasks(
5691            &backend,
5692            &tmux,
5693            "owner/repo",
5694            &runner,
5695            &capture,
5696            &semaphore,
5697            &task_manager,
5698            &weight_tx,
5699            dispatch_mode,
5700            &dispatching,
5701            &store,
5702            &HashMap::new(),
5703        )
5704        .await
5705        .unwrap();
5706
5707        let task = store.get(id).await.unwrap();
5708        assert_eq!(task.status, DbStatus::Waiting);
5709        assert!(dispatching.is_empty());
5710    }
5711
5712    /// Regression test for the completed-session -> stuck-task reclaim race.
5713    ///
5714    /// Scenario:
5715    /// - Runner finishes a tmux session and will call store_touch_updated_at shortly after
5716    /// - Engine's tick may observe the dead session in Phase 1 and Phase 2 may run
5717    ///   stuck-task recovery before the runner touches the store, causing incorrect reclaim
5718    /// Fix: tick_check_session_completions now touches the store.updated_at when it
5719    /// observes a completed session, preventing reclaim. This test simulates the
5720    /// timing by creating a session, marking the corresponding task InProgress with
5721    /// an old updated_at, calling tick_check_session_completions (which should touch
5722    /// the store), then calling tick_recover_stuck_tasks and asserting the task
5723    /// remains InProgress.
5724    #[tokio::test]
5725    async fn completed_session_does_not_cause_stuck_reclaim_race() {
5726        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5727        let mock = MockBackend::new();
5728        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5729        let task_manager = Arc::new(TaskManager::with_store(
5730            Arc::clone(&backend),
5731            Arc::clone(&store),
5732            "owner/repo".to_string(),
5733        ));
5734        let tmux = Arc::new(TmuxManager::new());
5735        let transport = Arc::new(Transport::new());
5736        let capture = Arc::new(CaptureService::new(transport));
5737
5738        // Insert external task and set to InProgress with old updated_at
5739        let id = store
5740            .upsert_external(&crate::store::UpsertExternal {
5741                repo: "owner/repo",
5742                ext_id: "200",
5743                title: "Race Task",
5744                body: "",
5745                author: "",
5746                url: "",
5747                labels: &[],
5748                origin: "github",
5749            })
5750            .await
5751            .unwrap();
5752        store
5753            .update_status(id, TaskStatus::InProgress)
5754            .await
5755            .unwrap();
5756
5757        // set updated_at to far past so it would normally be eligible for reclaim
5758        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5759            .bind(id)
5760            .execute(store.pool())
5761            .await
5762            .unwrap();
5763
5764        // Register a real tmux session for this task and then exit it so it's observed dead
5765        let task_id = "200";
5766        let session_name = tmux.session_name("owner/repo", task_id);
5767        let create_result = tmux
5768            .command()
5769            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5770            .output()
5771            .await;
5772        match create_result {
5773            Ok(o) if o.status.success() => {}
5774            _ => {
5775                eprintln!("Skipping test: tmux not available or failed to create test session");
5776                return;
5777            }
5778        }
5779        let _guard = SessionGuard(session_name.clone());
5780
5781        // Ensure remain-on-exit so the session lingers as dead long enough to be snapshot
5782        let _ = tmux
5783            .command()
5784            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5785            .output()
5786            .await;
5787        let _ = tmux
5788            .command()
5789            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5790            .output()
5791            .await;
5792
5793        // Poll until pane is dead
5794        for _ in 0..20 {
5795            time::sleep(Duration::from_millis(100)).await;
5796            if !tmux.session_is_running(&session_name).await {
5797                break;
5798            }
5799        }
5800
5801        // Register the session in capture (engine does this during dispatch)
5802        let task_key = task_id.to_string();
5803        capture
5804            .register_session("owner/repo", &task_key, &session_name)
5805            .await;
5806
5807        // Call phase1 (now touches store.updated_at for completed session)
5808        let config = EngineConfig {
5809            no_session_stuck_timeout: 600,
5810            stuck_timeout: 1800,
5811            ..EngineConfig::default()
5812        };
5813
5814        tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
5815            .await
5816            .unwrap();
5817
5818        // Verify phase1 actually touched updated_at. It only processes sessions
5819        // that appear in snapshot() (list_sessions + batch_session_active). On CI,
5820        // if the session was already fully removed before the snapshot runs (e.g.
5821        // remain-on-exit semantics differ across tmux versions), phase1 won't see
5822        // it and the store won't be touched. Skip in that case.
5823        {
5824            let task_state = store.get(id).await.unwrap();
5825            if task_state.updated_at.starts_with("2020") {
5826                eprintln!(
5827                    "Skipping test: tick_check_session_completions did not update stored \
5828                     updated_at — session was not in snapshot (CI tmux environment issue)"
5829                );
5830                return;
5831            }
5832        }
5833
5834        // Now run phase2 — the updated_at should have been touched so no reclaim occurs
5835        tick_recover_stuck_tasks(
5836            &backend,
5837            &tmux,
5838            "owner/repo",
5839            &task_manager,
5840            &config,
5841            &store,
5842            &HashMap::new(),
5843            &Arc::new(DashMap::new()),
5844        )
5845        .await
5846        .unwrap();
5847
5848        // Verify the task is still InProgress in the store
5849        let task = store.get(id).await.unwrap();
5850        assert_eq!(
5851            task.status,
5852            TaskStatus::InProgress,
5853            "task should not be reclaimed to New"
5854        );
5855    }
5856
5857    /// Regression test: a repo configured with a ".git" suffix (e.g. "owner/repo.git")
5858    /// must still match its own live tmux sessions in `tick_check_session_completions`.
5859    ///
5860    /// `TmuxManager::session_name` strips ".git" when creating the session, so a live
5861    /// session's parsed `project` field never carries it. Before the fix, this
5862    /// function's `repo_name` derivation skipped the `.trim_end_matches(".git")` step,
5863    /// so `session.project` ("repo") never equaled `repo_name` ("repo.git") and the
5864    /// completed-session touch/kill was silently skipped for the entire repo — leaving
5865    /// the coarser-grained stuck-task reclaim as the only (racy) safety net.
5866    #[tokio::test]
5867    async fn session_completion_matches_repo_configured_with_git_suffix() {
5868        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5869        let tmux = Arc::new(TmuxManager::new());
5870        let transport = Arc::new(Transport::new());
5871        let capture = Arc::new(CaptureService::new(transport));
5872
5873        let repo = "owner/repo.git";
5874        let id = store
5875            .upsert_external(&crate::store::UpsertExternal {
5876                repo,
5877                ext_id: "201",
5878                title: "Git-suffix repo task",
5879                body: "",
5880                author: "",
5881                url: "",
5882                labels: &[],
5883                origin: "github",
5884            })
5885            .await
5886            .unwrap();
5887        store
5888            .update_status(id, TaskStatus::InProgress)
5889            .await
5890            .unwrap();
5891        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5892            .bind(id)
5893            .execute(store.pool())
5894            .await
5895            .unwrap();
5896
5897        // session_name strips ".git" internally — this is the real session a dispatch
5898        // for this repo would have created.
5899        let task_id = "201";
5900        let session_name = tmux.session_name(repo, task_id);
5901        let create_result = tmux
5902            .command()
5903            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5904            .output()
5905            .await;
5906        match create_result {
5907            Ok(o) if o.status.success() => {}
5908            _ => {
5909                eprintln!("Skipping test: tmux not available or failed to create test session");
5910                return;
5911            }
5912        }
5913        let _guard = SessionGuard(session_name.clone());
5914
5915        let _ = tmux
5916            .command()
5917            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5918            .output()
5919            .await;
5920        let _ = tmux
5921            .command()
5922            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5923            .output()
5924            .await;
5925
5926        for _ in 0..20 {
5927            time::sleep(Duration::from_millis(100)).await;
5928            if !tmux.session_is_running(&session_name).await {
5929                break;
5930            }
5931        }
5932
5933        capture.register_session(repo, task_id, &session_name).await;
5934
5935        tick_check_session_completions(&tmux, repo, &capture, &store)
5936            .await
5937            .unwrap();
5938
5939        let task_state = store.get(id).await.unwrap();
5940        if task_state.updated_at.starts_with("2020") {
5941            eprintln!(
5942                "Skipping test: tick_check_session_completions did not update stored \
5943                 updated_at — session was not in snapshot (CI tmux environment issue)"
5944            );
5945            return;
5946        }
5947        assert!(
5948            !task_state.updated_at.starts_with("2020"),
5949            "tick_check_session_completions must match its own repo's live sessions \
5950             even when the repo is configured with a \".git\" suffix"
5951        );
5952    }
5953
5954    // ── tick_job_scheduler: resilience to bad project configs ────────────────
5955
5956    /// A project with a duplicate job id (same id in .orch.yml AND prompts/jobs/)
5957    /// must not cause tick_job_scheduler to return Err — it should log an error
5958    /// and return Ok(()) so other projects are unaffected.
5959    #[tokio::test]
5960    async fn job_scheduler_returns_ok_on_duplicate_job_config() {
5961        let tmp = tempfile::tempdir().unwrap();
5962        let cfg = tmp.path().join(".orch.yml");
5963        fs::write(
5964            &cfg,
5965            "jobs:\n  - id: trading-scan\n    schedule: \"0 9 * * *\"\n    task:\n      title: Scan\n      body: do it\n",
5966        )
5967        .unwrap();
5968        let jobs_dir = tmp.path().join("prompts").join("jobs");
5969        fs::create_dir_all(&jobs_dir).unwrap();
5970        fs::write(
5971            jobs_dir.join("trading-scan.md"),
5972            "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
5973        )
5974        .unwrap();
5975
5976        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5977        let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
5978        assert!(
5979            result.is_ok(),
5980            "duplicate job id must not propagate as Err: {result:?}"
5981        );
5982    }
5983
5984    /// When one project has a duplicate job id (bad config) and a second project
5985    /// has a valid config, the scheduler tick for the second project must still
5986    /// succeed — simulating the per-project loop in the engine.
5987    #[tokio::test]
5988    async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
5989        // Bad project: duplicate id across .orch.yml and prompts/jobs/
5990        let bad_tmp = tempfile::tempdir().unwrap();
5991        let bad_cfg = bad_tmp.path().join(".orch.yml");
5992        fs::write(
5993            &bad_cfg,
5994            "jobs:\n  - id: dup\n    schedule: \"0 9 * * *\"\n    task:\n      title: Dup\n      body: x\n",
5995        )
5996        .unwrap();
5997        let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
5998        fs::create_dir_all(&bad_jobs_dir).unwrap();
5999        fs::write(
6000            bad_jobs_dir.join("dup.md"),
6001            "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
6002        )
6003        .unwrap();
6004
6005        // Good project: valid config, no duplicate
6006        let good_tmp = tempfile::tempdir().unwrap();
6007        let good_cfg = good_tmp.path().join(".orch.yml");
6008        fs::write(
6009            &good_cfg,
6010            "jobs:\n  - id: valid-job\n    schedule: \"0 9 * * *\"\n    task:\n      title: Valid\n      body: ok\n",
6011        )
6012        .unwrap();
6013
6014        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
6015
6016        let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
6017        assert!(
6018            bad_result.is_ok(),
6019            "bad project must not abort scheduler: {bad_result:?}"
6020        );
6021
6022        let good_result =
6023            tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
6024        assert!(
6025            good_result.is_ok(),
6026            "valid project must still run after bad project: {good_result:?}"
6027        );
6028    }
6029
6030    // ── route_new_tasks_global ───────────────────────────────────────────────
6031
6032    /// Regression test for #3407: tasks from inactive/removed repos were stuck
6033    /// in `new` indefinitely because `tick_route_tasks` is repo-scoped.
6034    ///
6035    /// `route_new_tasks_global` must route tasks whose repo != current_repo
6036    /// and leave current-repo tasks untouched.
6037    #[serial_test::serial(cooldown_state)]
6038    #[tokio::test]
6039    async fn global_sweep_routes_inactive_repo_tasks() {
6040        cooldown::reset_global_state().await;
6041
6042        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6043
6044        // Active repo task — must NOT be touched by the global sweep.
6045        let active_id = store
6046            .upsert_external(&UpsertExternal {
6047                repo: "active/repo",
6048                ext_id: "1",
6049                title: "Active task",
6050                body: "",
6051                author: "user",
6052                url: "https://github.com/active/repo/issues/1",
6053                labels: &[],
6054                origin: "github",
6055            })
6056            .await
6057            .unwrap();
6058
6059        // Inactive repo task — must be routed by the global sweep.
6060        let inactive_id = store
6061            .upsert_external(&UpsertExternal {
6062                repo: "inactive/repo",
6063                ext_id: "42",
6064                title: "Orphaned task",
6065                body: "",
6066                author: "user",
6067                url: "https://github.com/inactive/repo/issues/42",
6068                labels: &[],
6069                origin: "github",
6070            })
6071            .await
6072            .unwrap();
6073
6074        assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
6075        assert_eq!(
6076            store.get(inactive_id).await.unwrap().status,
6077            TaskStatus::New
6078        );
6079
6080        let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6081        let mut tier = HashMap::new();
6082        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6083        model_map.insert("medium".to_string(), tier);
6084        let config = RouterConfig {
6085            mode: "round_robin".to_string(),
6086            agents: vec!["claude".to_string()],
6087            model_map,
6088            ..Default::default()
6089        };
6090        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6091
6092        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6093        let mut routing_quota = RoutingTickQuota::new(1);
6094        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6095            .await
6096            .unwrap();
6097
6098        // Active-repo task must remain new (global sweep skips it).
6099        assert_eq!(
6100            store.get(active_id).await.unwrap().status,
6101            TaskStatus::New,
6102            "global sweep must not touch tasks for the active repo"
6103        );
6104
6105        // Inactive-repo task must now be routed.
6106        let after = store.get(inactive_id).await.unwrap();
6107        assert_eq!(
6108            after.status,
6109            TaskStatus::Routed,
6110            "global sweep must route tasks from inactive repos"
6111        );
6112        assert!(
6113            matches!(&after.agent, Some(a) if !a.is_empty()),
6114            "agent must be set after routing"
6115        );
6116    }
6117
6118    /// Regression test for #3523: `route_new_tasks_global` previously filtered only
6119    /// `t.repo != current_repo`, so during repo A's tick it would route repo B's `new`
6120    /// tasks too — even though B is an active, non-orphaned project with its own tick
6121    /// scheduled later in the same pass. This is the exact "racing the real per-repo
6122    /// dispatch" failure mode `dispatch_routed_tasks_global` was fixed for in #3415;
6123    /// the route-side sweep needs the same `active_repos` scoping, not just the
6124    /// current tick's own repo.
6125    #[serial_test::serial(cooldown_state)]
6126    #[tokio::test]
6127    async fn global_sweep_does_not_touch_other_active_repo_tasks() {
6128        cooldown::reset_global_state().await;
6129
6130        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6131
6132        // Belongs to a second active repo — NOT the repo whose tick is running,
6133        // but still a currently-configured project, not an orphan.
6134        let other_active_id = store
6135            .upsert_external(&UpsertExternal {
6136                repo: "other-active/repo",
6137                ext_id: "7",
6138                title: "Other active project's task",
6139                body: "",
6140                author: "user",
6141                url: "https://github.com/other-active/repo/issues/7",
6142                labels: &[],
6143                origin: "github",
6144            })
6145            .await
6146            .unwrap();
6147
6148        // Genuinely orphaned repo — not in the active set at all.
6149        let inactive_id = store
6150            .upsert_external(&UpsertExternal {
6151                repo: "inactive/repo",
6152                ext_id: "42",
6153                title: "Orphaned task",
6154                body: "",
6155                author: "user",
6156                url: "https://github.com/inactive/repo/issues/42",
6157                labels: &[],
6158                origin: "github",
6159            })
6160            .await
6161            .unwrap();
6162
6163        let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6164        let mut tier = HashMap::new();
6165        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6166        model_map.insert("medium".to_string(), tier);
6167        let config = RouterConfig {
6168            mode: "round_robin".to_string(),
6169            agents: vec!["claude".to_string()],
6170            model_map,
6171            ..Default::default()
6172        };
6173        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6174
6175        // Simulate "active/repo"'s tick running the global sweep. Both
6176        // "active/repo" and "other-active/repo" are currently-configured
6177        // projects; only "inactive/repo" is orphaned.
6178        let active_repos: HashSet<String> =
6179            ["active/repo".to_string(), "other-active/repo".to_string()]
6180                .into_iter()
6181                .collect();
6182        let mut routing_quota = RoutingTickQuota::new(1);
6183        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6184            .await
6185            .unwrap();
6186
6187        assert_eq!(
6188            store.get(other_active_id).await.unwrap().status,
6189            TaskStatus::New,
6190            "global sweep run from a different active repo's tick must not route \
6191             another active repo's new tasks — that repo's own tick will route them"
6192        );
6193        assert_eq!(
6194            store.get(inactive_id).await.unwrap().status,
6195            TaskStatus::Routed,
6196            "global sweep must still route tasks from genuinely orphaned repos"
6197        );
6198    }
6199
6200    /// Tasks with the `no-agent` label must be skipped by the global sweep.
6201    #[serial_test::serial(cooldown_state)]
6202    #[tokio::test]
6203    async fn global_sweep_skips_no_agent_tasks() {
6204        cooldown::reset_global_state().await;
6205
6206        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6207
6208        let id = store
6209            .upsert_external(&UpsertExternal {
6210                repo: "inactive/repo",
6211                ext_id: "99",
6212                title: "No-agent task",
6213                body: "",
6214                author: "user",
6215                url: "https://github.com/inactive/repo/issues/99",
6216                labels: &["no-agent".to_string()],
6217                origin: "github",
6218            })
6219            .await
6220            .unwrap();
6221
6222        let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6223        let mut tier = HashMap::new();
6224        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6225        model_map.insert("medium".to_string(), tier);
6226        let config = RouterConfig {
6227            mode: "round_robin".to_string(),
6228            agents: vec!["claude".to_string()],
6229            model_map,
6230            ..Default::default()
6231        };
6232        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6233
6234        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6235        let mut routing_quota = RoutingTickQuota::new(1);
6236        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6237            .await
6238            .unwrap();
6239
6240        assert_eq!(
6241            store.get(id).await.unwrap().status,
6242            TaskStatus::New,
6243            "no-agent tasks must not be routed"
6244        );
6245    }
6246
6247    /// Regression test for #3615: the routing limit must be consumed by a
6248    /// shared per-engine-tick quota, not independently by every repo/global
6249    /// routing pass. Otherwise one slow router timeout per pass can stack into
6250    /// a multi-minute watchdog stall.
6251    #[serial_test::serial(cooldown_state)]
6252    #[tokio::test]
6253    async fn global_sweep_honors_shared_routing_quota() {
6254        cooldown::reset_global_state().await;
6255
6256        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6257        for ext_id in ["1", "2"] {
6258            store
6259                .upsert_external(&UpsertExternal {
6260                    repo: "inactive/repo",
6261                    ext_id,
6262                    title: "Orphaned task",
6263                    body: "",
6264                    author: "user",
6265                    url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
6266                    labels: &[],
6267                    origin: "github",
6268                })
6269                .await
6270                .unwrap();
6271        }
6272
6273        let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6274        let mut tier = HashMap::new();
6275        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6276        model_map.insert("medium".to_string(), tier);
6277        let config = RouterConfig {
6278            mode: "round_robin".to_string(),
6279            agents: vec!["claude".to_string()],
6280            model_map,
6281            ..Default::default()
6282        };
6283        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6284
6285        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6286        let mut routing_quota = RoutingTickQuota::new(1);
6287        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6288            .await
6289            .unwrap();
6290
6291        let all_tasks = store.list_all_global().await.unwrap();
6292        let routed = all_tasks
6293            .iter()
6294            .filter(|task| task.status == TaskStatus::Routed)
6295            .count();
6296        let still_new = all_tasks
6297            .iter()
6298            .filter(|task| task.status == TaskStatus::New)
6299            .count();
6300
6301        assert_eq!(routed, 1, "only one task may consume the shared quota");
6302        assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
6303
6304        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6305            .await
6306            .unwrap();
6307        let after_exhausted = store.list_all_global().await.unwrap();
6308        assert_eq!(
6309            after_exhausted
6310                .iter()
6311                .filter(|task| task.status == TaskStatus::Routed)
6312                .count(),
6313            1,
6314            "an exhausted shared quota must not reset within the same tick"
6315        );
6316    }
6317
6318    /// A task of a repo with no project directory ends `blocked` with a reason that names
6319    /// the cause, never `needs_review` (#4024)
6320    #[tokio::test]
6321    async fn block_task_without_project_blocks_with_the_cause() {
6322        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6323        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
6324        let task_manager = Arc::new(TaskManager::with_store(
6325            backend,
6326            Arc::clone(&store),
6327            "removed/repo".to_string(),
6328        ));
6329        let id = store
6330            .create_internal("removed/repo", "Orphan", "", "cron", "1", None)
6331            .await
6332            .unwrap();
6333        store.update_status(id, TaskStatus::Routed).await.unwrap();
6334
6335        block_task_without_project(
6336            &task_manager,
6337            &store,
6338            "removed/repo",
6339            &format!("internal:{id}"),
6340        )
6341        .await;
6342
6343        let task = store.get(id).await.unwrap();
6344        assert_eq!(task.status, TaskStatus::Blocked);
6345        assert_eq!(
6346            task.block_reason.as_deref(),
6347            Some(NoProjectDir::BLOCK_REASON)
6348        );
6349    }
6350
6351    // ── dispatch_routed_tasks_global ─────────────────────────────────────────
6352    //
6353    // `dispatch_routed_tasks_global` itself is deliberately not exercised
6354    // end-to-end here: once it finds an orphaned repo, it spawns the real
6355    // runner, which creates a real tmux session and touches the network.
6356    // The existing `dispatch_does_not_deadlock_under_write_lock` test avoids
6357    // this same hazard by keeping the dispatchable set empty. The selection
6358    // logic (which tasks belong to which repo) is pulled into the pure
6359    // `group_orphaned_routed_tasks` helper below so it can be tested without
6360    // those side effects.
6361
6362    /// Regression test for #3413: tasks routed for an inactive/removed repo were
6363    /// stuck in `routed` indefinitely because `tick_dispatch_tasks` is repo-scoped.
6364    ///
6365    /// `group_orphaned_routed_tasks` must select tasks whose repo != the active
6366    /// repo, group them by repo, and leave active-repo tasks out entirely.
6367    #[tokio::test]
6368    async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
6369        async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6370            let id = store
6371                .upsert_external(&UpsertExternal {
6372                    repo,
6373                    ext_id,
6374                    title: "Routed task",
6375                    body: "",
6376                    author: "user",
6377                    url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6378                    labels: &[],
6379                    origin: "github",
6380                })
6381                .await
6382                .unwrap();
6383            store
6384                .store_route(&StoreRoute {
6385                    id,
6386                    agent: "claude",
6387                    model: Some("sonnet"),
6388                    complexity: "medium",
6389                    estimate: 0,
6390                    reason: "test",
6391                    profile: "{}",
6392                    skills: "[]",
6393                })
6394                .await
6395                .unwrap();
6396            store.update_status(id, TaskStatus::Routed).await.unwrap();
6397            id
6398        }
6399
6400        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6401
6402        let active_id = make_routed(&store, "active/repo", "1").await;
6403        let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
6404        let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
6405        let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
6406
6407        let all_routed = store
6408            .list_all_by_status_global(TaskStatus::Routed)
6409            .await
6410            .unwrap();
6411        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6412        let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6413
6414        assert!(
6415            !by_repo.contains_key("active/repo"),
6416            "global sweep must not select tasks for the active repo"
6417        );
6418        let inactive_ids: HashSet<i64> = by_repo
6419            .get("inactive/repo")
6420            .expect("inactive/repo must be selected")
6421            .iter()
6422            .map(|t| t.id)
6423            .collect();
6424        assert_eq!(
6425            inactive_ids,
6426            [inactive_id_a, inactive_id_b].into_iter().collect(),
6427            "global sweep must group all orphaned tasks for a repo together"
6428        );
6429        assert_eq!(
6430            by_repo
6431                .get("other-inactive/repo")
6432                .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6433            Some(vec![other_inactive_id])
6434        );
6435
6436        // Sanity: the active-repo task really was created and routed, it's just
6437        // excluded from the selection above.
6438        assert_eq!(
6439            store.get(active_id).await.unwrap().status,
6440            TaskStatus::Routed
6441        );
6442    }
6443
6444    /// Regression test for the PR #3413 review finding: a single `current_repo` string is
6445    /// the wrong exclusion — repo B's routed tasks must not look "orphaned" during repo A's
6446    /// dispatch just because A != B.
6447    /// `group_orphaned_routed_tasks` must exclude every repo in the full active set.
6448    #[tokio::test]
6449    async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
6450        async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6451            let id = store
6452                .upsert_external(&UpsertExternal {
6453                    repo,
6454                    ext_id,
6455                    title: "Routed task",
6456                    body: "",
6457                    author: "user",
6458                    url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6459                    labels: &[],
6460                    origin: "github",
6461                })
6462                .await
6463                .unwrap();
6464            store
6465                .store_route(&StoreRoute {
6466                    id,
6467                    agent: "claude",
6468                    model: Some("sonnet"),
6469                    complexity: "medium",
6470                    estimate: 0,
6471                    reason: "test",
6472                    profile: "{}",
6473                    skills: "[]",
6474                })
6475                .await
6476                .unwrap();
6477            store.update_status(id, TaskStatus::Routed).await.unwrap();
6478            id
6479        }
6480
6481        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6482
6483        let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
6484        let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
6485        let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
6486
6487        let all_routed = store
6488            .list_all_by_status_global(TaskStatus::Routed)
6489            .await
6490            .unwrap();
6491        // Simulate repo A's tick: current repo is "active/repo-a", but the full
6492        // active project set also includes "active/repo-b".
6493        let active_repos: HashSet<String> =
6494            ["active/repo-a".to_string(), "active/repo-b".to_string()]
6495                .into_iter()
6496                .collect();
6497        let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6498
6499        assert!(
6500            !by_repo.contains_key("active/repo-a"),
6501            "the current repo's own tasks must not be swept"
6502        );
6503        assert!(
6504            !by_repo.contains_key("active/repo-b"),
6505            "another active repo's tasks must not be swept during repo A's tick"
6506        );
6507        assert_eq!(
6508            by_repo
6509                .get("inactive/repo")
6510                .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6511            Some(vec![orphaned_id]),
6512            "a genuinely inactive repo's tasks must still be swept"
6513        );
6514
6515        // Sanity: repo A and repo B tasks were really created and routed, just excluded.
6516        assert_eq!(
6517            store.get(repo_a_id).await.unwrap().status,
6518            TaskStatus::Routed
6519        );
6520        assert_eq!(
6521            store.get(repo_b_id).await.unwrap().status,
6522            TaskStatus::Routed
6523        );
6524    }
6525
6526    /// Tasks with the `no-agent` label must be skipped by the global dispatch sweep.
6527    #[tokio::test]
6528    async fn global_dispatch_sweep_skips_no_agent_tasks() {
6529        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6530
6531        let id = store
6532            .upsert_external(&UpsertExternal {
6533                repo: "inactive/repo",
6534                ext_id: "99",
6535                title: "No-agent routed task",
6536                body: "",
6537                author: "user",
6538                url: "https://github.com/inactive/repo/issues/99",
6539                labels: &["no-agent".to_string()],
6540                origin: "github",
6541            })
6542            .await
6543            .unwrap();
6544        store
6545            .store_route(&StoreRoute {
6546                id,
6547                agent: "claude",
6548                model: Some("sonnet"),
6549                complexity: "medium",
6550                estimate: 0,
6551                reason: "test",
6552                profile: "{}",
6553                skills: "[]",
6554            })
6555            .await
6556            .unwrap();
6557        store.update_status(id, TaskStatus::Routed).await.unwrap();
6558
6559        // Safe to call the full sweep here: with the only orphaned task filtered
6560        // out by the no-agent label, `group_orphaned_routed_tasks` returns an
6561        // empty map and `dispatch_routed_tasks_global` returns before ever
6562        // constructing a backend or spawning a runner.
6563        let tmux = Arc::new(TmuxManager::new());
6564        let transport = Arc::new(Transport::new());
6565        let capture = Arc::new(CaptureService::new(transport));
6566        let semaphore = Arc::new(Semaphore::new(4));
6567        let (weight_tx, _weight_rx) = mpsc::channel(16);
6568        let dispatching = Arc::new(DashMap::new());
6569        let dispatch_mode = DispatchMode {
6570            is_degraded: false,
6571            healthy_agents: 1,
6572            threshold: 1,
6573        };
6574
6575        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6576        dispatch_routed_tasks_global(
6577            &tmux,
6578            &capture,
6579            &semaphore,
6580            &weight_tx,
6581            dispatch_mode,
6582            &dispatching,
6583            &store,
6584            &HashMap::new(),
6585            &active_repos,
6586        )
6587        .await
6588        .unwrap();
6589
6590        assert_eq!(
6591            store.get(id).await.unwrap().status,
6592            TaskStatus::Routed,
6593            "no-agent tasks must not be dispatched"
6594        );
6595    }
6596}