Skip to main content

orch/engine/
tick.rs

1//! Core tick loop phases.
2//!
3//! The engine ticks every ~10 seconds. Each tick runs six phases:
4//! 1. Poll tmux for finished sessions
5//! 2. Recover stuck in-progress tasks
6//! 3. Route and dispatch:
7//!    - 3a. Route new tasks to agents
8//!    - 3b. Dispatch routed tasks (spawn agents)
9//! 4. Unblock parents whose children are all done
10//! 5. Run cron job scheduler
11//!
12//! [`tick_all`] runs them for every project. Phases 1-2 and 3b-5 run for all projects at
13//! once; only routing is serial, because it holds the router write lock. Store-wide work
14//! (`tick_global_sweeps`, `reconcile_dependencies`) runs once per tick, not once per project.
15
16use super::{EngineConfig, ProjectEngine};
17use crate::backends::github::GitHubBackend;
18use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
19use crate::channels::capture::CaptureService;
20use crate::channels::transport::Transport;
21use crate::cmd_cache;
22use crate::config;
23use crate::engine;
24use crate::engine::cooldown::{
25    SILENCE_AGENT_COOLDOWN_SECS, SILENCE_EXTENDED_COOLDOWN_SECS, github_circuit_remaining_secs,
26    is_agent_in_cooldown, is_github_circuit_open, is_model_in_cooldown,
27    record_agent_failure_with_message, record_silence_detection, set_agent_cooldown,
28    set_model_cooldown,
29};
30use crate::engine::dependencies;
31use crate::engine::dispatch_guard::DispatchGuard;
32use crate::engine::jobs;
33use crate::engine::priority;
34use crate::engine::priority::Priority;
35use crate::engine::router::config as router_config;
36use crate::engine::router::{
37    AllAgentsCooledError, RouteResultError, Router, RouterConfig, get_route_result,
38};
39use crate::engine::runner::response;
40use crate::engine::runner::{NoProjectDir, TaskRunner, WeightSignal};
41use crate::engine::sync;
42use crate::engine::tasks;
43use crate::engine::tasks::{TaskManager, is_internal_id};
44use crate::github::http;
45use crate::repo_context::REPO_CONTEXT;
46use crate::store;
47use crate::store::StoreRoute;
48use crate::store::Task;
49use crate::store::TaskStatus;
50use crate::store::TaskStatus as DbStatus;
51use crate::store::TaskStore;
52use crate::store::WaitingOnDependency;
53use crate::store::{store_set_by_id, store_touch_updated_at};
54use crate::tmux::TmuxManager;
55use dashmap::mapref::entry::Entry;
56use dashmap::{DashMap, DashSet};
57use futures::future;
58use std::collections::HashMap;
59use std::collections::HashSet;
60use std::path::Path;
61use std::sync::Arc;
62use std::sync::OnceLock;
63use std::time::Duration;
64use std::time::{Instant, SystemTime};
65use tokio::sync::RwLock;
66use tokio::sync::{Semaphore, mpsc};
67use tokio::time;
68
69/// Per outer engine-loop quota for expensive routing calls.
70///
71/// Routing runs once per active project. Without a quota shared by all of them,
72/// `router.max_tasks_per_tick = 1` still permits one slow LLM routing call per
73/// project, multiplying 45s router timeouts into multi-minute watchdog stalls.
74pub(crate) struct RoutingTickQuota {
75    remaining: usize,
76}
77
78impl RoutingTickQuota {
79    pub(crate) fn new(max: usize) -> Self {
80        Self { remaining: max }
81    }
82
83    fn is_exhausted(&self) -> bool {
84        self.remaining == 0
85    }
86
87    fn remaining(&self) -> usize {
88        self.remaining
89    }
90
91    fn consume_one(&mut self) -> bool {
92        if self.remaining == 0 {
93            return false;
94        }
95        self.remaining -= 1;
96        true
97    }
98}
99
100/// Cleanup stale tmux sessions from previous engine runs.
101/// Called once from `serve()` before the main loop, never from a tick
102pub(crate) async fn startup_cleanup(tmux: &TmuxManager) {
103    let start_time = SystemTime::now()
104        .duration_since(SystemTime::UNIX_EPOCH)
105        .unwrap_or_default()
106        .as_secs();
107    tracing::info!(
108        start_time,
109        "running startup cleanup for stale tmux sessions"
110    );
111
112    match tmux.kill_stale_sessions(start_time).await {
113        Ok(killed) => {
114            if killed > 0 {
115                tracing::info!(killed, "startup cleanup killed stale sessions");
116            } else {
117                tracing::debug!("no stale tmux sessions found on startup");
118            }
119        }
120        Err(e) => {
121            tracing::warn!(error = %e, "failed to run startup cleanup");
122        }
123    }
124}
125
126async fn recover_routed_blocked_dispatch(
127    backend: &Arc<dyn ExternalBackend>,
128    store: &Arc<TaskStore>,
129    task_manager: &Arc<TaskManager>,
130    repo: &str,
131    task: &ExternalTask,
132    timing: &StuckTaskTiming,
133) {
134    tracing::warn!(
135        task_id = task.id.0,
136        age_mins = timing.age.num_minutes(),
137        threshold_mins = timing.threshold / 60,
138        "recovering routed task: dispatch blocked by stale tmux session → new"
139    );
140
141    let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
142        Ok(Some(id)) => Some(id),
143        Ok(None) => {
144            tracing::warn!(
145                task_id = %task.id.0,
146                repo,
147                "resolve_task_id returned None during routed-task recovery"
148            );
149            None
150        }
151        Err(e) => {
152            tracing::error!(
153                task_id = %task.id.0,
154                repo,
155                error = %e,
156                "resolve_task_id failed during routed-task recovery"
157            );
158            None
159        }
160    };
161
162    if let Some(store_id) = resolved_store_id {
163        if let Err(e) = store_set_by_id(
164            Some(store),
165            store_id,
166            &[
167                ("agent", serde_json::Value::Null),
168                ("model", serde_json::Value::Null),
169                ("route_attempts", serde_json::json!(0)),
170            ],
171        )
172        .await
173        {
174            tracing::warn!(
175                task_id = task.id.0,
176                ?e,
177                "failed to clear routing fields for routed stuck task — will retry next tick"
178            );
179            return;
180        }
181        finalize_open_runs_for_recovery(
182            store,
183            store_id,
184            "stuck-task recovery: stale session blocking dispatch killed",
185        )
186        .await;
187    }
188
189    if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
190        tracing::warn!(
191            task_id = task.id.0,
192            ?e,
193            "failed to reset routed stuck task status"
194        );
195        return;
196    }
197
198    store::store_log_activity(
199        &Some(Arc::clone(store)),
200        repo,
201        &task.id.0,
202        "rerouted",
203        Some("routed"),
204        Some("new"),
205        None::<&str>,
206        None::<&str>,
207        Some(&serde_json::json!({
208            "failure_reason": "routed_blocked_by_existing_session",
209            "age_minutes": timing.age.num_minutes(),
210            "had_session": true,
211        })),
212    )
213    .await;
214
215    if !is_internal_id(&task.id.0) {
216        let backend_clone = Arc::clone(backend);
217        let task_id_clone = task.id.clone();
218        let body = format!(
219            "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
220            chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
221            timing.age.num_minutes(),
222            engine::orch_footer()
223        );
224        tokio::spawn(async move {
225            if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
226                tracing::warn!(
227                    task_id = task_id_clone.0,
228                    ?e,
229                    "failed to post routed-task recovery comment"
230                );
231            }
232        });
233    }
234}
235
236/// Phase 1 of tick: poll tmux for finished sessions and clean them up.
237pub(crate) async fn tick_check_session_completions(
238    tmux: &Arc<TmuxManager>,
239    repo: &str,
240    capture: &Arc<CaptureService>,
241    store: &Arc<TaskStore>,
242) -> anyhow::Result<()> {
243    let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
244    let session_snapshot = tmux.snapshot().await;
245    // Derive the short project name from repo (owner/repo -> repo), matching
246    // TmuxManager::session_name's derivation exactly (including the ".git" strip)
247    // so a repo configured with a ".git" suffix doesn't silently fail to match
248    // its own live sessions here — leaving stuck-task recovery as the only
249    // (later, coarser-grained) safety net for this repo's session completions.
250    let repo_name = repo
251        .rsplit('/')
252        .next()
253        .unwrap_or(repo)
254        .trim_end_matches(".git");
255    for (session, active) in session_snapshot {
256        // Only handle sessions that belong to this repo/project.
257        if session.project != repo_name {
258            continue;
259        }
260
261        if !active {
262            tracing::info!(
263                session = %session.name,
264                task_id = %session.task_id,
265                "session completed, collecting results"
266            );
267            // Touch the store.updated_at immediately when we observe a completed
268            // session so the stuck-task recovery phase doesn't incorrectly
269            // reclaim this task while the runner is still finishing
270            // post-processing (race: session observed dead → tick reclaim).
271            // Best-effort: no-ops if the task isn't present in the store.
272            //
273            // session.task_id uses hyphens (e.g. "internal-63714") because the
274            // tmux session name sanitizes colons to hyphens. resolve_task_id only
275            // handles "internal:" prefix, so we must convert back before touching.
276            // Review sessions carry a "-review" suffix tracked under the main task.
277            let store_task_id = {
278                let base = session
279                    .task_id
280                    .strip_suffix("-review")
281                    .unwrap_or(&session.task_id);
282                if let Some(n) = base.strip_prefix("internal-") {
283                    format!("internal:{n}")
284                } else {
285                    base.to_string()
286                }
287            };
288            store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
289            // Unregister from capture service using the task id the capture service
290            // was registered under (task id without project prefix).
291            capture.unregister_session(repo, &session.task_id).await;
292            // Kill the actual tmux session name we discovered (do not reconstruct)
293            if let Err(e) = tmux.kill_session(&session.name).await {
294                tracing::debug!(
295                    session = %session.name,
296                    ?e,
297                    "kill_session failed (session may already be gone)"
298                );
299            }
300        }
301    }
302    Ok(())
303}
304
305#[derive(Debug)]
306struct StuckTaskTiming {
307    has_session: bool,
308    age: chrono::Duration,
309    threshold: u64,
310}
311
312fn stuck_task_timing_from_map(
313    tmux: &Arc<TmuxManager>,
314    repo: &str,
315    session_task_id: &str,
316    updated_at: &str,
317    config: &EngineConfig,
318    parse_error_message: &'static str,
319    session_map: &HashMap<String, bool>,
320) -> Option<StuckTaskTiming> {
321    let has_session = session_alive(tmux, repo, session_task_id, session_map);
322    let threshold = if has_session {
323        config.stuck_timeout
324    } else {
325        config.no_session_stuck_timeout
326    };
327
328    let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
329        Ok(dt) => dt.with_timezone(&chrono::Utc),
330        Err(e) => {
331            tracing::warn!(
332                task_id = session_task_id,
333                updated_at,
334                ?e,
335                error_message = parse_error_message,
336                "cannot parse updated_at"
337            );
338            return None;
339        }
340    };
341
342    let raw_age = chrono::Utc::now() - updated;
343    // Host suspend (laptop sleep) freezes the process just like it freezes the agent
344    // running inside it — that frozen time is not evidence of a hang. Discount any
345    // detected suspend/resume gaps that occurred since this task was last touched.
346    let suspended = super::suspend::suspended_duration_since(updated);
347    let age = (raw_age - suspended).max(chrono::Duration::zero());
348    if age.num_seconds() <= i64::try_from(threshold).unwrap_or(i64::MAX) {
349        return None;
350    }
351
352    Some(StuckTaskTiming {
353        has_session,
354        age,
355        threshold,
356    })
357}
358
359fn session_alive(
360    tmux: &TmuxManager,
361    repo: &str,
362    session_task_id: &str,
363    session_map: &HashMap<String, bool>,
364) -> bool {
365    session_map
366        .get(&tmux.session_name(repo, session_task_id))
367        .copied()
368        .unwrap_or(false)
369}
370
371/// The tier whose run timeout the task runs with: the tier of its stored model, which differs from
372/// its own complexity when the model came from an adjacent tier
373fn run_tier(router: &RouterConfig, task: &Task) -> String {
374    match (task.agent.as_deref(), task.model.as_deref()) {
375        (Some(a), Some(m)) if !m.is_empty() => router.tier_of_model(a, m, &task.complexity),
376        _ => task.complexity.clone(),
377    }
378}
379
380/// The router config for this tick, loaded on first use. A full load re-reads the config and
381/// repeats its startup warnings, so one tick shares one load across every live run
382fn tick_router(
383    cell: &OnceLock<RouterConfig>,
384    load: impl FnOnce() -> RouterConfig,
385) -> &RouterConfig {
386    cell.get_or_init(load)
387}
388
389/// Start time and config for the stuck check of an in_progress task. A live session ages from
390/// the status entry, since other writes refresh `updated_at` mid-run, and is not stuck before
391/// its own run timeout ends. A dead session keeps `updated_at`, which the runner touches when it
392/// sees the session end
393async fn in_progress_stuck_clock(
394    store: &TaskStore,
395    repo: &str,
396    task_id: &str,
397    updated_at: &str,
398    has_session: bool,
399    config: &EngineConfig,
400    router: &OnceLock<RouterConfig>,
401) -> (String, EngineConfig) {
402    if !has_session {
403        return (updated_at.to_string(), config.clone());
404    }
405
406    let task = match store.resolve_task_id(repo, task_id).await {
407        Ok(Some(store_id)) => store.get(store_id).await.ok(),
408        _ => None,
409    };
410    let Some(task) = task else {
411        return (updated_at.to_string(), config.clone());
412    };
413
414    let entered_at = match store
415        .status_entered_at(task.id, TaskStatus::InProgress)
416        .await
417    {
418        Ok(Some(ts)) => ts,
419        Ok(None) => updated_at.to_string(),
420        Err(e) => {
421            tracing::warn!(
422                task_id,
423                error = %e,
424                "failed to read in_progress entry, falling back to updated_at"
425            );
426            updated_at.to_string()
427        }
428    };
429    let tier = run_tier(tick_router(router, RouterConfig::from_config), &task);
430    let task_config = EngineConfig {
431        stuck_timeout: config.live_session_stuck_limit(Some(&tier)),
432        ..config.clone()
433    };
434    (entered_at, task_config)
435}
436
437/// When the task last entered in_review, and whether a review decision landed since. Age from the
438/// status entry: `review_open_prs` rewrites `updated_at` on every pass, so it never ages
439async fn in_review_stuck_entry(store: &TaskStore, store_id: i64, fallback: &str) -> (String, bool) {
440    match store.in_review_entry(store_id).await {
441        Ok(Some(entry)) => entry,
442        Ok(None) => (fallback.to_string(), false),
443        Err(e) => {
444            tracing::warn!(
445                store_id,
446                error = %e,
447                "failed to read in_review entry, falling back to updated_at"
448            );
449            (fallback.to_string(), false)
450        }
451    }
452}
453
454/// Finalize any open `task_runs` rows for a task during recovery paths.
455/// Best-effort: logs a warning on failure but does not block recovery.
456async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
457    if let Err(e) = store
458        .finalize_incomplete_runs(store_id, "aborted", reason)
459        .await
460    {
461        tracing::warn!(
462            store_id,
463            error = %e,
464            reason,
465            "failed to finalize incomplete runs during recovery"
466        );
467    }
468}
469
470/// Phase 1b of tick: detect agents that have produced no output since session start
471/// and have exceeded the silence grace period.
472///
473/// Only triggers on complete silence (agent never produced any output). Agents that
474/// produced output then went quiet (e.g. long tool calls) are NOT affected.
475///
476/// On silence: kills the tmux session, applies model-level cooldown, resets the task
477/// to `New` so the router picks a different agent/model.
478pub(crate) async fn tick_detect_silent_agents(
479    tmux: &Arc<TmuxManager>,
480    repo: &str,
481    capture: &Arc<CaptureService>,
482    backend: &Arc<dyn ExternalBackend>,
483    task_manager: &Arc<TaskManager>,
484    config: &EngineConfig,
485    store: &Arc<TaskStore>,
486) -> anyhow::Result<()> {
487    let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
488    let grace = Duration::from_secs(config.silence_grace_period);
489    let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
490
491    for (task_id, session_name, silence_age_secs) in silent_sessions {
492        let use_backend = should_use_backend(&task_id);
493        // Look up agent + model from the store so we can cooldown the right model.
494        let store_task = match store.resolve_task_id(repo, &task_id).await {
495            Ok(Some(store_id)) => match store.get(store_id).await {
496                Ok(task) => Some(task),
497                Err(e) => {
498                    tracing::warn!(
499                        task_id,
500                        repo,
501                        store_id,
502                        operation = "store.get",
503                        error = %e,
504                        "silence detection: db error fetching task — agent/model cooldown will be skipped"
505                    );
506                    None
507                }
508            },
509            Ok(None) => {
510                tracing::debug!(
511                    task_id,
512                    repo,
513                    operation = "resolve_task_id",
514                    "silence detection: task not found in store (may have been cleaned up)"
515                );
516                None
517            }
518            Err(e) => {
519                tracing::warn!(
520                    task_id,
521                    repo,
522                    operation = "resolve_task_id",
523                    error = %e,
524                    "silence detection: db error resolving task id — agent/model cooldown will be skipped"
525                );
526                None
527            }
528        };
529        let agent_name = store_task
530            .as_ref()
531            .and_then(|t| t.agent.clone())
532            .unwrap_or_default();
533        let model_name = store_task
534            .as_ref()
535            .and_then(|t| t.model.clone())
536            .unwrap_or_default();
537
538        tracing::warn!(
539            task_id,
540            agent = %agent_name,
541            model = %model_name,
542            silence_age_secs,
543            grace_secs = config.silence_grace_period,
544            cooldown_secs = config.silence_cooldown,
545            "agent silent since session start — killing session, cooling down model + agent, failing over"
546        );
547
548        // 1. Kill the tmux session
549        if let Err(e) = tmux.kill_session(&session_name).await {
550            tracing::debug!(
551                task_id,
552                ?e,
553                "kill_session failed for silent agent (may already be gone)"
554            );
555        }
556
557        // 2. Unregister from capture
558        capture.unregister_session(repo, &task_id).await;
559
560        // Finalize any open audit run so the task_runs row is not orphaned
561        if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
562            finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
563                .await;
564        }
565
566        let mut extended_note = String::new();
567
568        // 3. Cooldown the specific model (not the whole agent)
569        if !agent_name.is_empty() && !model_name.is_empty() {
570            set_model_cooldown(
571                &agent_name,
572                &model_name,
573                config.silence_cooldown,
574                "silence_detected",
575            )
576            .await;
577            if let Some(result) = record_silence_detection(&agent_name, &model_name).await
578                && result.extended_cooldown_applied
579            {
580                extended_note = format!(
581                    " ({} silences in 24h -> extended cooldown {}s)",
582                    result.count, SILENCE_EXTENDED_COOLDOWN_SECS
583                );
584            }
585        }
586
587        // 3b. Short agent-level cooldown to force router to pick a different agent.
588        // Without this, the router picks the same agent with a different model,
589        // looping through all models (~2 min each) before the long agent cooldown kicks in.
590        if !agent_name.is_empty() {
591            set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
592        }
593
594        // 4. Pick a fallback agent and set to Routed (not New) to preserve progress.
595        // Setting to New would clear routing state and trigger a full LLM re-routing
596        // cycle, losing intermediate context. Routed skips re-routing and re-dispatches
597        // directly with the chosen fallback agent.
598        let task_eid = ExternalId(task_id.clone());
599
600        // Build available agents list and reroute chain for failover
601        let available: Vec<String> = engine::configured_agents()
602            .into_iter()
603            .filter(|a| cmd_cache::command_exists(a))
604            .collect();
605        let chain = response::get_reroute_chain(&task_id, &Some(Arc::clone(store)), repo).await;
606        let chain = response::update_reroute_chain(
607            &task_id,
608            &agent_name,
609            &chain,
610            &Some(Arc::clone(store)),
611            repo,
612        )
613        .await;
614
615        let (next_status, next_agent) = if let Some(fallback) =
616            response::pick_fallback_agent(&agent_name, &chain, &available)
617        {
618            tracing::info!(
619                task_id,
620                from = %agent_name,
621                to = %fallback,
622                "silence detection: failover to different agent, setting routed"
623            );
624            (Status::Routed, Some(fallback))
625        } else {
626            tracing::warn!(
627                task_id,
628                "silence detection: no fallback agents available, resetting to new for re-routing"
629            );
630            (Status::New, None)
631        };
632
633        if use_backend && let Some(ref st) = store_task {
634            // Fire-and-forget: label removals are cosmetic — tick should not block on them.
635            let stale_labels: Vec<String> = st
636                .labels
637                .iter()
638                .filter(|l| {
639                    l.starts_with("agent:")
640                        || l.starts_with("complexity:")
641                        || l.starts_with("model:")
642                })
643                .cloned()
644                .collect();
645            if !stale_labels.is_empty() {
646                let backend_clone = Arc::clone(backend);
647                let task_eid_clone = task_eid.clone();
648                tokio::spawn(async move {
649                    for label in &stale_labels {
650                        if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await {
651                            tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
652                        }
653                    }
654                });
655            }
656        }
657
658        if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
659            if let Some(ref fallback) = next_agent {
660                // Write the fallback agent into the store so dispatch can proceed
661                // directly without an LLM re-routing cycle. Clearing model forces
662                // model_for_complexity to pick the best available model for the new agent.
663                let _ = store_set_by_id(
664                    Some(store),
665                    store_id,
666                    &[
667                        ("agent", serde_json::json!(fallback)),
668                        ("model", serde_json::json!("")),
669                        (
670                            "last_error",
671                            serde_json::json!(format!(
672                                "silence detected after {}s, failing over to {}",
673                                config.silence_grace_period, fallback
674                            )),
675                        ),
676                    ],
677                )
678                .await;
679            } else {
680                let _ = store_set_by_id(
681                    Some(store),
682                    store_id,
683                    &[
684                        ("agent", serde_json::Value::Null),
685                        ("model", serde_json::Value::Null),
686                        (
687                            "last_error",
688                            serde_json::json!(format!(
689                                "silence detected after {}s, no fallback agents available",
690                                config.silence_grace_period
691                            )),
692                        ),
693                    ],
694                )
695                .await;
696            }
697        }
698
699        if let Err(e) = task_manager
700            .update_task_status(&task_eid, next_status)
701            .await
702        {
703            tracing::warn!(task_id, ?e, "failed to update silent task status");
704            continue;
705        }
706
707        // Log reroute activity
708        let details = serde_json::json!({
709            "failure_reason": "agent_silence",
710            "silence_duration_secs": config.silence_grace_period,
711            "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
712            "fallback_available": next_agent.is_some(),
713        });
714        store::store_log_activity(
715            &Some(Arc::clone(store)),
716            repo,
717            &task_id,
718            "rerouted",
719            Some("in_progress"),
720            Some(next_status.as_str()),
721            next_agent.as_deref(),
722            None,
723            Some(&details),
724        )
725        .await;
726
727        // 5. Post a comment explaining what happened
728        let action = if let Some(ref fallback) = next_agent {
729            format!("failing over to {fallback}")
730        } else {
731            "marking needs_review (no fallback agents)".to_string()
732        };
733        let comment = format!(
734            "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
735            chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
736            config.silence_grace_period,
737            agent_name,
738            model_name,
739            config.silence_cooldown,
740            extended_note,
741            agent_name,
742            SILENCE_AGENT_COOLDOWN_SECS,
743            action,
744            engine::orch_footer(),
745        );
746        if use_backend {
747            // Fire-and-forget: comment is informational — tick should not block on it.
748            let backend_clone = Arc::clone(backend);
749            let task_eid_clone = task_eid.clone();
750            tokio::spawn(async move {
751                if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
752                    tracing::warn!(
753                        task_id = task_eid_clone.0,
754                        ?e,
755                        "failed to post silence detection comment"
756                    );
757                }
758            });
759        }
760    }
761
762    Ok(())
763}
764
765fn should_use_backend(task_id: &str) -> bool {
766    !is_internal_id(task_id)
767}
768
769/// Phase 2 of tick: recover tasks stranded by missing or stale tmux state.
770/// - `in_progress` tasks → reset to `New` when their session is missing or the run timed out
771/// - `routed` tasks      → reset to `New` when an old live session keeps blocking re-dispatch
772/// - `in_review` tasks   → reset to `NeedsReview` when the review session disappeared
773#[expect(
774    clippy::too_many_arguments,
775    reason = "each argument is engine state the tick passes down"
776)]
777pub(crate) async fn tick_recover_stuck_tasks(
778    backend: &Arc<dyn ExternalBackend>,
779    tmux: &Arc<TmuxManager>,
780    repo: &str,
781    task_manager: &Arc<TaskManager>,
782    config: &EngineConfig,
783    store: &Arc<TaskStore>,
784    session_map: &HashMap<String, bool>,
785    dispatching: &Arc<DashMap<String, String>>,
786) -> anyhow::Result<()> {
787    let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
788    let tick_router_cell = OnceLock::new();
789
790    let in_progress = match task_manager
791        .list_external_by_status(Status::InProgress)
792        .await
793    {
794        Ok(tasks) => tasks,
795        Err(e) => {
796            tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
797            vec![]
798        }
799    };
800    for task in &in_progress {
801        let (since, task_config) = in_progress_stuck_clock(
802            store,
803            repo,
804            &task.id.0,
805            &task.updated_at,
806            session_alive(tmux, repo, &task.id.0, session_map),
807            config,
808            &tick_router_cell,
809        )
810        .await;
811        let Some(timing) = stuck_task_timing_from_map(
812            tmux,
813            repo,
814            &task.id.0,
815            &since,
816            &task_config,
817            "cannot parse updated_at, skipping stuck-task check",
818            session_map,
819        ) else {
820            continue;
821        };
822
823        // A dead tmux session with no `updated_at` refresh in a long time normally means
824        // the runner vanished (crashed, orch restarted). But it also matches the tail end
825        // of a *successful* run: the session exits the instant the agent finishes, while
826        // the runner is still polling for that exit (up to its own poll interval) and then
827        // writing results. If the runner's dispatch guard for this task is still held, the
828        // runner is alive and actively processing — let it finish rather than racing it.
829        if !timing.has_session {
830            let dispatch_key = format!("{repo}/{}", task.id.0);
831            if dispatching.contains_key(&dispatch_key) {
832                // Promoted from debug! to info!: the default log filter is "orch=info"
833                // (see main.rs), so this line — the only direct evidence that the guard
834                // check actually found and honored a held dispatch guard — was silently
835                // dropped in production. Every prior occurrence of the reclaim race
836                // (#3518/#3519/#3523/#3525/#3526) had to be diagnosed from timestamp
837                // correlation alone because this and the "dispatch guard
838                // claimed"/"released" lines never appeared in orch.log.
839                tracing::info!(
840                    task_id = task.id.0,
841                    dispatch_key,
842                    age_mins = timing.age.num_minutes(),
843                    "skipping stuck-task reclaim: runner still actively processing this task"
844                );
845                continue;
846            }
847        }
848
849        if timing.has_session {
850            tracing::warn!(
851                task_id = task.id.0,
852                age_mins = timing.age.num_minutes(),
853                threshold_mins = timing.threshold / 60,
854                "recovering stuck task: timed out with active session → new"
855            );
856            // The agent would keep working under a task that is routed again, and its session
857            // name would block the next dispatch
858            let session_name = tmux.session_name(repo, &task.id.0);
859            if let Err(e) = tmux.kill_session(&session_name).await {
860                tracing::warn!(
861                    task_id = task.id.0,
862                    session = %session_name,
863                    error = %e,
864                    "failed to kill stuck tmux session, will retry next tick"
865                );
866                continue;
867            }
868        } else {
869            tracing::warn!(
870                task_id = task.id.0,
871                dispatch_key = format!("{repo}/{}", task.id.0),
872                age_mins = timing.age.num_minutes(),
873                threshold_mins = timing.threshold / 60,
874                "recovering stuck task: no session found — reclaiming early → new"
875            );
876        }
877        // For stuck tasks with an active session, record agent/model failure + cooldown so the
878        // router picks a different agent/model on the next attempt. Without this, the router
879        // sees the agent as healthy and selects it again, causing the hang to repeat.
880        // (Mirrors the pattern in tick_detect_silent_agents lines 219–284.)
881        let mut cached_store_id: Option<i64> = None;
882        if timing.has_session {
883            let store_task = match store.resolve_task_id(repo, &task.id.0).await {
884                Ok(Some(store_id)) => {
885                    cached_store_id = Some(store_id);
886                    match store.get(store_id).await {
887                        Ok(t) => Some(t),
888                        Err(e) => {
889                            tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
890                            None
891                        }
892                    }
893                }
894                Ok(None) => None,
895                Err(e) => {
896                    tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
897                    None
898                }
899            };
900            let agent_name = store_task
901                .as_ref()
902                .and_then(|t| t.agent.clone())
903                .unwrap_or_default();
904            let model_name = store_task
905                .as_ref()
906                .and_then(|t| t.model.clone())
907                .unwrap_or_default();
908
909            if !agent_name.is_empty() && !model_name.is_empty() {
910                set_model_cooldown(
911                    &agent_name,
912                    &model_name,
913                    config.silence_cooldown,
914                    "silence_detected",
915                )
916                .await;
917                record_agent_failure_with_message(
918                    &agent_name,
919                    &format!(
920                        "stuck with active session after {}m",
921                        timing.age.num_minutes()
922                    ),
923                )
924                .await;
925            }
926            if !agent_name.is_empty() {
927                set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
928            }
929            tracing::warn!(
930                task_id = task.id.0,
931                agent = %agent_name,
932                model = %model_name,
933                cooldown_secs = config.silence_cooldown,
934                "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
935            );
936        }
937
938        // Remove stale agent/model labels so the LLM router re-routes properly.
939        // Fire-and-forget: cosmetic label operation, tick should not block on it.
940        {
941            let stale_labels: Vec<String> = task
942                .labels
943                .iter()
944                .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
945                .cloned()
946                .collect();
947            if !stale_labels.is_empty() {
948                let backend_clone = Arc::clone(backend);
949                let task_id_clone = task.id.clone();
950                tokio::spawn(async move {
951                    for label in &stale_labels {
952                        if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
953                            tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
954                        }
955                    }
956                });
957            }
958        }
959        let resolved_store_id = match cached_store_id {
960            Some(id) => Some(id),
961            None => match store.resolve_task_id(repo, &task.id.0).await {
962                Ok(Some(id)) => Some(id),
963                Ok(None) => {
964                    tracing::warn!(
965                        task_id = %task.id.0,
966                        repo,
967                        "resolve_task_id returned None during stuck-task recovery"
968                    );
969                    None
970                }
971                Err(e) => {
972                    tracing::error!(
973                        task_id = %task.id.0,
974                        repo,
975                        error = %e,
976                        "resolve_task_id failed during stuck-task recovery"
977                    );
978                    None
979                }
980            },
981        };
982        if let Some(store_id) = resolved_store_id {
983            if let Err(e) = store_set_by_id(
984                Some(store),
985                store_id,
986                &[
987                    ("agent", serde_json::Value::Null),
988                    ("model", serde_json::Value::Null),
989                    ("route_attempts", serde_json::json!(0)),
990                ],
991            )
992            .await
993            {
994                tracing::warn!(
995                    task_id = task.id.0,
996                    ?e,
997                    "failed to clear routing fields for stuck task — will retry next tick"
998                );
999                continue;
1000            }
1001            finalize_open_runs_for_recovery(
1002                store,
1003                store_id,
1004                if timing.has_session {
1005                    "stuck-task recovery: active session killed"
1006                } else {
1007                    "stuck-task recovery: no session found"
1008                },
1009            )
1010            .await;
1011        }
1012        if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
1013            tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
1014            continue;
1015        }
1016
1017        // Log reroute activity for stuck task recovery
1018        let details = serde_json::json!({
1019            "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1020            "age_minutes": timing.age.num_minutes(),
1021            "had_session": timing.has_session,
1022        });
1023        store::store_log_activity(
1024            &Some(Arc::clone(store)),
1025            repo,
1026            &task.id.0,
1027            "rerouted",
1028            Some("in_progress"),
1029            Some("new"),
1030            None::<&str>,
1031            None::<&str>,
1032            Some(&details),
1033        )
1034        .await;
1035
1036        let reason = if timing.has_session {
1037            format!(
1038                "timed out after {}m with active session (cleared agent for re-routing)",
1039                timing.age.num_minutes()
1040            )
1041        } else {
1042            format!(
1043                "no session found — reclaiming early after {}m (cleared agent for re-routing)",
1044                timing.age.num_minutes()
1045            )
1046        };
1047        // Fire-and-forget: comment is informational — tick should not block on it.
1048        {
1049            let backend_clone = Arc::clone(backend);
1050            let task_id_clone = task.id.clone();
1051            let body = format!(
1052                "[{}] recovered: stuck in_progress — {}{}",
1053                chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1054                reason,
1055                engine::orch_footer()
1056            );
1057            tokio::spawn(async move {
1058                if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
1059                    tracing::warn!(
1060                        task_id = task_id_clone.0,
1061                        ?e,
1062                        "failed to post stuck-task recovery comment"
1063                    );
1064                }
1065            });
1066        }
1067    }
1068
1069    let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
1070        Ok(tasks) => tasks,
1071        Err(e) => {
1072            tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
1073            vec![]
1074        }
1075    };
1076    match task_manager
1077        .list_internal_by_status(TaskStatus::Routed)
1078        .await
1079    {
1080        Ok(tasks) => routed.extend(tasks),
1081        Err(e) => tracing::warn!(
1082            ?e,
1083            "failed to list internal routed tasks for stuck recovery"
1084        ),
1085    }
1086    for task in &routed {
1087        let Some(timing) = stuck_task_timing_from_map(
1088            tmux,
1089            repo,
1090            &task.id.0,
1091            &task.updated_at,
1092            config,
1093            "cannot parse updated_at, skipping routed stuck-task check",
1094            session_map,
1095        ) else {
1096            continue;
1097        };
1098
1099        if !timing.has_session {
1100            continue;
1101        }
1102
1103        let session_name = tmux.session_name(repo, &task.id.0);
1104        if let Err(e) = tmux.kill_session(&session_name).await {
1105            tracing::warn!(
1106                task_id = task.id.0,
1107                session = %session_name,
1108                error = %e,
1109                "failed to kill stale tmux session blocking routed task"
1110            );
1111            continue;
1112        }
1113
1114        recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
1115    }
1116
1117    // Recover internal (SQLite) tasks stuck in in_progress.
1118    // These have no GitHub labels or comments — just reset the DB status to New.
1119    let internal_in_progress = match task_manager
1120        .list_internal_by_status(DbStatus::InProgress)
1121        .await
1122    {
1123        Ok(tasks) => tasks,
1124        Err(e) => {
1125            tracing::warn!(
1126                ?e,
1127                "failed to list internal in_progress tasks for stuck recovery"
1128            );
1129            vec![]
1130        }
1131    };
1132    for task in &internal_in_progress {
1133        let task_id = task.id.0.clone();
1134        let (since, task_config) = in_progress_stuck_clock(
1135            store,
1136            repo,
1137            &task_id,
1138            &task.updated_at,
1139            session_alive(tmux, repo, &task_id, session_map),
1140            config,
1141            &tick_router_cell,
1142        )
1143        .await;
1144        let Some(timing) = stuck_task_timing_from_map(
1145            tmux,
1146            repo,
1147            &task_id,
1148            &since,
1149            &task_config,
1150            "cannot parse updated_at, skipping stuck internal-task check",
1151            session_map,
1152        ) else {
1153            continue;
1154        };
1155
1156        // See the comment on the external in-progress loop above: a dead session with a
1157        // stale `updated_at` also matches the tail end of a run that just finished
1158        // successfully. Skip the reclaim while the runner's dispatch guard for this task
1159        // is still held — this is the exact race that discards completed daily/scheduled
1160        // internal tasks whose total runtime exceeds `no_session_stuck_timeout`.
1161        if !timing.has_session {
1162            let dispatch_key = format!("{repo}/{task_id}");
1163            if dispatching.contains_key(&dispatch_key) {
1164                // Promoted from debug! to info! (issue #3526): the default log filter is
1165                // "orch=info" (main.rs), so this line — along with the DispatchGuard
1166                // claim/release lines in dispatch_guard.rs and dispatch_tasks_for_repo —
1167                // never reached orch.log in production. Every occurrence of the reclaim
1168                // race (#3518, #3519, #3523, #3525, #3526) had to be reasoned about from
1169                // timestamp correlation across unrelated log lines instead of directly
1170                // observing whether the guard was held, because the one log line that
1171                // would prove it was silently filtered out. Promoting these closes that
1172                // observability gap so the next occurrence carries conclusive evidence.
1173                tracing::info!(
1174                    task_id,
1175                    dispatch_key,
1176                    age_mins = timing.age.num_minutes(),
1177                    "skipping stuck-task reclaim: runner still actively processing this task"
1178                );
1179                continue;
1180            }
1181        }
1182
1183        if timing.has_session {
1184            tracing::warn!(
1185                task_id,
1186                age_mins = timing.age.num_minutes(),
1187                threshold_mins = timing.threshold / 60,
1188                "recovering stuck task: timed out with active session → new"
1189            );
1190            let session_name = tmux.session_name(repo, &task_id);
1191            if let Err(e) = tmux.kill_session(&session_name).await {
1192                tracing::warn!(
1193                    task_id,
1194                    session = %session_name,
1195                    error = %e,
1196                    "failed to kill stuck tmux session, will retry next tick"
1197                );
1198                continue;
1199            }
1200        } else {
1201            tracing::warn!(
1202                task_id,
1203                dispatch_key = format!("{repo}/{task_id}"),
1204                age_mins = timing.age.num_minutes(),
1205                threshold_mins = timing.threshold / 60,
1206                "recovering stuck task: no session found — reclaiming early → new"
1207            );
1208        }
1209        // Apply agent/model cooldown for internal stuck tasks with active sessions,
1210        // mirroring the external path (lines 501-546). Without this, the router picks
1211        // the same agent/model that caused the hang, creating an infinite loop.
1212        let mut cached_store_id: Option<i64> = None;
1213        if timing.has_session {
1214            let store_task = match store.resolve_task_id(repo, &task_id).await {
1215                Ok(Some(store_id)) => {
1216                    cached_store_id = Some(store_id);
1217                    match store.get(store_id).await {
1218                        Ok(t) => Some(t),
1219                        Err(e) => {
1220                            tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1221                            None
1222                        }
1223                    }
1224                }
1225                Ok(None) => None,
1226                Err(e) => {
1227                    tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1228                    None
1229                }
1230            };
1231            let agent_name = store_task
1232                .as_ref()
1233                .and_then(|t| t.agent.clone())
1234                .unwrap_or_default();
1235            let model_name = store_task
1236                .as_ref()
1237                .and_then(|t| t.model.clone())
1238                .unwrap_or_default();
1239
1240            if !agent_name.is_empty() && !model_name.is_empty() {
1241                set_model_cooldown(
1242                    &agent_name,
1243                    &model_name,
1244                    config.silence_cooldown,
1245                    "silence_detected",
1246                )
1247                .await;
1248                record_agent_failure_with_message(
1249                    &agent_name,
1250                    &format!(
1251                        "internal task stuck with active session after {}m",
1252                        timing.age.num_minutes()
1253                    ),
1254                )
1255                .await;
1256            }
1257            if !agent_name.is_empty() {
1258                set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1259            }
1260            tracing::warn!(
1261                task_id,
1262                agent = %agent_name,
1263                model = %model_name,
1264                cooldown_secs = config.silence_cooldown,
1265                "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1266            );
1267        }
1268        // Reset routing state so the LLM router is used on the next attempt
1269        // (same reset that external tasks perform).
1270        let resolved_store_id = match cached_store_id {
1271            Some(id) => Some(id),
1272            None => match store.resolve_task_id(repo, &task_id).await {
1273                Ok(Some(id)) => Some(id),
1274                Ok(None) => {
1275                    tracing::warn!(
1276                        task_id = %task_id,
1277                        repo,
1278                        "resolve_task_id returned None during stuck internal-task recovery"
1279                    );
1280                    None
1281                }
1282                Err(e) => {
1283                    tracing::error!(
1284                        task_id = %task_id,
1285                        repo,
1286                        error = %e,
1287                        "resolve_task_id failed during stuck internal-task recovery"
1288                    );
1289                    None
1290                }
1291            },
1292        };
1293        if let Some(store_id) = resolved_store_id {
1294            if let Err(e) = store_set_by_id(
1295                Some(store),
1296                store_id,
1297                &[
1298                    ("agent", serde_json::Value::Null),
1299                    ("model", serde_json::Value::Null),
1300                    ("route_attempts", serde_json::json!(0)),
1301                ],
1302            )
1303            .await
1304            {
1305                tracing::warn!(
1306                    task_id,
1307                    ?e,
1308                    "failed to clear routing fields for stuck internal task — will retry next tick"
1309                );
1310                continue;
1311            }
1312            finalize_open_runs_for_recovery(
1313                store,
1314                store_id,
1315                if timing.has_session {
1316                    "stuck-task recovery: internal active session killed"
1317                } else {
1318                    "stuck-task recovery: internal no session found"
1319                },
1320            )
1321            .await;
1322        }
1323        if let Err(e) = task_manager
1324            .update_task_status(&ExternalId(task_id.clone()), Status::New)
1325            .await
1326        {
1327            tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1328        }
1329    }
1330
1331    // Recover tasks stuck in in_review, external and internal in one sweep. Reset to NeedsReview
1332    // (not New) so the review agent re-triggers without clearing the routing state.
1333    //
1334    // This is the only periodic owner of the case where the review session is gone. The sync
1335    // sweep used to reset the same tasks after one minute, which beat a finished review's result
1336    // to the engine and dropped it as stale, and left this longer limit unreachable
1337    let in_review = match store.list_by_status(repo, TaskStatus::InReview).await {
1338        Ok(v) => v,
1339        Err(e) => {
1340            tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1341            vec![]
1342        }
1343    };
1344    // A review agent ends its tmux session before its result reaches the engine, so the
1345    // no-session limit for a review has to be longer than for a task agent. The floor in
1346    // `EngineConfig::from_config` keeps it above the review agent's own timeout
1347    let in_review_config = engine::EngineConfig {
1348        no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1349        ..config.clone()
1350    };
1351    for task in &in_review {
1352        let Some(task_id) = task.external_id.clone() else {
1353            tracing::warn!(
1354                store_id = task.id,
1355                "in_review task has no external id, skipping stuck recovery"
1356            );
1357            continue;
1358        };
1359
1360        // The dispatch flow owns a task while it holds the guard
1361        if dispatching.contains_key(&format!("{repo}/{task_id}")) {
1362            tracing::debug!(
1363                task_id,
1364                "task locked by dispatch flow, skipping stuck check"
1365            );
1366            continue;
1367        }
1368
1369        // A recorded decision means the review finished and the task waits on the merge, not on
1370        // a review agent
1371        let (entered_at, decided) = in_review_stuck_entry(store, task.id, &task.updated_at).await;
1372        if decided {
1373            tracing::debug!(
1374                task_id,
1375                "in_review task already has a review decision, waiting on merge"
1376            );
1377            continue;
1378        }
1379
1380        let review_task_id = format!("{task_id}-review");
1381        let Some(timing) = stuck_task_timing_from_map(
1382            tmux,
1383            repo,
1384            &review_task_id,
1385            &entered_at,
1386            &in_review_config,
1387            "cannot parse in_review timestamp, skipping stuck in_review check",
1388            session_map,
1389        ) else {
1390            continue;
1391        };
1392
1393        tracing::warn!(
1394            task_id,
1395            age_mins = timing.age.num_minutes(),
1396            threshold_mins = timing.threshold / 60,
1397            has_session = timing.has_session,
1398            "recovering stuck in_review task — resetting to needs_review"
1399        );
1400
1401        match task_manager
1402            .update_task_status_if(
1403                &ExternalId(task_id.clone()),
1404                Status::NeedsReview,
1405                Status::InReview,
1406            )
1407            .await
1408        {
1409            Err(e) => {
1410                tracing::warn!(task_id, ?e, "failed to reset stuck in_review task status");
1411                continue;
1412            }
1413            Ok(false) => {
1414                tracing::debug!(
1415                    task_id,
1416                    "stuck in_review reset skipped — task already transitioned"
1417                );
1418                continue;
1419            }
1420            Ok(true) => {}
1421        }
1422
1423        // A dead or hung session is an infrastructure event, a tmux crash or a restart, not an
1424        // agent parse failure. Keeping the counter would spend the next cycle's quota
1425        if let Err(e) = store
1426            .set_fields(task.id, &[("review_agent_failures", serde_json::json!(0))])
1427            .await
1428        {
1429            tracing::warn!(task_id, ?e, "failed to reset review_agent_failures");
1430        }
1431
1432        finalize_open_runs_for_recovery(
1433            store,
1434            task.id,
1435            if timing.has_session {
1436                "stuck-task recovery: in_review session killed"
1437            } else {
1438                "stuck-task recovery: in_review no session found"
1439            },
1440        )
1441        .await;
1442
1443        let details = serde_json::json!({
1444            "failure_reason": "stuck_in_review",
1445            "age_minutes": timing.age.num_minutes(),
1446            "had_session": timing.has_session,
1447        });
1448        store::store_log_activity(
1449            &Some(Arc::clone(store)),
1450            repo,
1451            &task_id,
1452            "rerouted",
1453            Some("in_review"),
1454            Some("needs_review"),
1455            None::<&str>,
1456            None::<&str>,
1457            Some(&details),
1458        )
1459        .await;
1460
1461        // An internal task has no issue to comment on
1462        if task.origin == "internal" {
1463            continue;
1464        }
1465
1466        if let Err(e) = backend
1467            .post_comment(
1468                &ExternalId(task_id.clone()),
1469                &format!(
1470                    "[{}] recovered: stuck in_review — {} after {}m, resetting to needs_review{}",
1471                    chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1472                    if timing.has_session {
1473                        "review session hung"
1474                    } else {
1475                        "no session found"
1476                    },
1477                    timing.age.num_minutes(),
1478                    engine::orch_footer()
1479                ),
1480            )
1481            .await
1482        {
1483            tracing::warn!(
1484                task_id,
1485                ?e,
1486                "failed to post stuck in_review recovery comment"
1487            );
1488        }
1489    }
1490
1491    Ok(())
1492}
1493
1494/// External tasks stuck in_progress in a repo no project ticks, such as a project removed from
1495/// the config. Store-wide, so `tick_all` runs it once per tick rather than once per project
1496pub(crate) async fn recover_untracked_repo_stuck_tasks(
1497    tmux: &Arc<TmuxManager>,
1498    config: &EngineConfig,
1499    store: &Arc<TaskStore>,
1500    session_map: &HashMap<String, bool>,
1501    dispatching: &Arc<DashMap<String, String>>,
1502    active_repos: &HashSet<String>,
1503) {
1504    let all_external_in_progress = match store
1505        .list_all_by_status_global(TaskStatus::InProgress)
1506        .await
1507    {
1508        Ok(tasks) => tasks,
1509        Err(e) => {
1510            tracing::warn!(
1511                ?e,
1512                "failed to list global in_progress tasks for cross-repo stuck recovery"
1513            );
1514            vec![]
1515        }
1516    };
1517    for task in all_external_in_progress
1518        .iter()
1519        .filter(|t| t.origin != "internal" && !active_repos.contains(&t.repo))
1520    {
1521        let external_id = task
1522            .external_id
1523            .clone()
1524            .unwrap_or_else(|| task.id.to_string());
1525        let Some(timing) = stuck_task_timing_from_map(
1526            tmux,
1527            &task.repo,
1528            &external_id,
1529            &task.updated_at,
1530            config,
1531            "cannot parse updated_at, skipping cross-repo stuck-task check",
1532            session_map,
1533        ) else {
1534            continue;
1535        };
1536
1537        // See the comment on the in-progress loop in `tick_recover_stuck_tasks`: a dead session with a stale
1538        // `updated_at` also matches the tail end of a run that just finished successfully.
1539        // Skip the reclaim while the runner's dispatch guard for this task is still held.
1540        if !timing.has_session {
1541            let dispatch_key = format!("{}/{external_id}", task.repo);
1542            if dispatching.contains_key(&dispatch_key) {
1543                // See the same promotion note on the internal-task loop in `tick_recover_stuck_tasks`: this was
1544                // debug! (silently dropped under the default "orch=info" filter), leaving
1545                // every prior reclaim-race occurrence undiagnosable from logs alone.
1546                tracing::info!(
1547                    task_id = task.id,
1548                    repo = %task.repo,
1549                    dispatch_key,
1550                    age_mins = timing.age.num_minutes(),
1551                    "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1552                );
1553                continue;
1554            }
1555        }
1556
1557        if timing.has_session {
1558            tracing::warn!(
1559                task_id = task.id,
1560                repo = %task.repo,
1561                age_mins = timing.age.num_minutes(),
1562                threshold_mins = timing.threshold / 60,
1563                "recovering cross-repo stuck task: timed out with active session → new"
1564            );
1565        } else {
1566            tracing::warn!(
1567                task_id = task.id,
1568                repo = %task.repo,
1569                dispatch_key = format!("{}/{external_id}", task.repo),
1570                age_mins = timing.age.num_minutes(),
1571                threshold_mins = timing.threshold / 60,
1572                "recovering cross-repo stuck task: no session found — reclaiming early → new"
1573            );
1574        }
1575
1576        if let Err(e) = store_set_by_id(
1577            Some(store),
1578            task.id,
1579            &[
1580                ("agent", serde_json::Value::Null),
1581                ("model", serde_json::Value::Null),
1582                ("route_attempts", serde_json::json!(0)),
1583            ],
1584        )
1585        .await
1586        {
1587            tracing::warn!(
1588                task_id = task.id,
1589                repo = %task.repo,
1590                ?e,
1591                "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1592            );
1593            continue;
1594        }
1595
1596        finalize_open_runs_for_recovery(
1597            store,
1598            task.id,
1599            if timing.has_session {
1600                "stuck-task recovery: cross-repo active session killed"
1601            } else {
1602                "stuck-task recovery: cross-repo no session found"
1603            },
1604        )
1605        .await;
1606
1607        if let Err(e) = store.update_status(task.id, TaskStatus::New).await {
1608            tracing::warn!(
1609                task_id = task.id,
1610                repo = %task.repo,
1611                ?e,
1612                "failed to reset cross-repo stuck task status"
1613            );
1614            continue;
1615        }
1616
1617        let details = serde_json::json!({
1618            "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1619            "age_minutes": timing.age.num_minutes(),
1620            "had_session": timing.has_session,
1621            "scope": "cross_repo_global_sweep",
1622        });
1623        store::store_log_activity(
1624            &Some(Arc::clone(store)),
1625            &task.repo,
1626            &external_id,
1627            "rerouted",
1628            Some("in_progress"),
1629            Some("new"),
1630            None::<&str>,
1631            None::<&str>,
1632            Some(&details),
1633        )
1634        .await;
1635    }
1636}
1637
1638/**
1639 * The three store-wide recovery sweeps, run once per tick rather than once per project.
1640 *
1641 * Each one reads every repo, so a per-project call duplicates the work. The NeedsReview
1642 * sweep also reads `needs_review_refires` and then increments it, so concurrent per-project
1643 * passes all see the same count and four projects burn four of the five refire attempts in
1644 * one tick (#3864)
1645 **/
1646pub(crate) async fn tick_global_sweeps(
1647    task_manager: &Arc<TaskManager>,
1648    store: &Arc<TaskStore>,
1649    router: &Router,
1650    dispatching: &Arc<DashMap<String, String>>,
1651    auto_merge_in_flight: &Arc<DashSet<String>>,
1652    active_repos: &HashSet<String>,
1653) {
1654    // Global sweep for CI-failure blocked tasks from inactive or removed projects.
1655    // auto_unblock_blocked_tasks runs inside sync_tick and is scoped to the active repo.
1656    // Tasks blocked for projects no longer in orch config are never processed there.
1657    // This sweep covers all repos so stale CI-failure blocks are eventually resolved.
1658    if let Err(e) = sync::auto_unblock_ci_failure_blocked_tasks_global(
1659        task_manager,
1660        store,
1661        auto_merge_in_flight,
1662    )
1663    .await
1664    {
1665        tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1666    }
1667
1668    // Re-fire stale NeedsReview tasks and escalate persistently-stuck ones to
1669    // Blocked, across all repos
1670    if let Err(e) = sync::refire_and_escalate_stale_needs_review_global(
1671        task_manager,
1672        store,
1673        router,
1674        dispatching,
1675        active_repos,
1676    )
1677    .await
1678    {
1679        tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1680    }
1681
1682    // Recover rebroadcast-blocked tasks once a review agent is routable, across
1683    // all repos
1684    if let Err(e) = sync::auto_recover_rebroadcast_blocked_tasks_global(
1685        router,
1686        task_manager,
1687        store,
1688        active_repos,
1689    )
1690    .await
1691    {
1692        tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1693    }
1694}
1695
1696/// Phase 3a of tick: route status:new tasks to an agent and transition them to status:routed.
1697pub(crate) async fn tick_route_tasks(
1698    backend: &Arc<dyn ExternalBackend>,
1699    task_manager: &Arc<TaskManager>,
1700    router: &mut Router,
1701    store: &Arc<TaskStore>,
1702    repo: &str,
1703    routing_quota: &mut RoutingTickQuota,
1704) -> anyhow::Result<()> {
1705    let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1706    if let Err(err) = dependencies::apply_claim_gates(store).await {
1707        tracing::warn!(err = %err, "dependency claim gate failed");
1708    }
1709    if routing_quota.is_exhausted() {
1710        tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1711        return Ok(());
1712    }
1713
1714    // Global GitHub 5xx circuit breaker — skip routing during sustained GitHub outages
1715    // to avoid routing-heavy retry storms. Tasks will remain in 'new' status and be
1716    // retried when the circuit closes.
1717    if is_github_circuit_open() {
1718        let remaining = github_circuit_remaining_secs();
1719        tracing::info!(
1720            remaining_secs = remaining,
1721            "GitHub 5xx circuit breaker open — skipping routing phase"
1722        );
1723        return Ok(());
1724    }
1725    let new_tasks = task_manager.list_routable().await?;
1726    let routable: Vec<&ExternalTask> = new_tasks
1727        .iter()
1728        .filter(|t| !skips_agent(&t.labels))
1729        .collect();
1730
1731    // Pre-emptive health check: refresh degraded-agent flags once per tick,
1732    // not per-task (previously this ran inside route() causing N redundant
1733    // DB queries when routing N tasks in the same tick).
1734    if !routable.is_empty() {
1735        let start = Instant::now();
1736        router.refresh_health(store).await;
1737        tracing::debug!(
1738            duration_ms = start.elapsed().as_millis(),
1739            "router.refresh_health completed"
1740        );
1741    }
1742
1743    // Limit routing to the remaining global quota for this outer engine tick.
1744    for task in routable.into_iter().take(routing_quota.remaining()) {
1745        if !routing_quota.consume_one() {
1746            break;
1747        }
1748        let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1749
1750        let task_start = Instant::now();
1751        match router.route(task, store, repo).await {
1752            Ok(result) => {
1753                tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1754                // Store route result in store
1755                if let Err(e) = router
1756                    .store_route_result(&task.id.0, &result, store, repo)
1757                    .await
1758                {
1759                    tracing::warn!(
1760                        task_id = task.id.0,
1761                        ?e,
1762                        "failed to store route result; skipping Routed transition"
1763                    );
1764                    continue;
1765                }
1766
1767                if is_internal_id(&task.id.0) {
1768                    // Internal tasks: update DB status, skip GitHub label ops.
1769                    if let Err(e) = task_manager
1770                        .update_task_status(&task.id, Status::Routed)
1771                        .await
1772                    {
1773                        tracing::warn!(
1774                            task_id = task.id.0,
1775                            ?e,
1776                            "failed to set internal status:routed"
1777                        );
1778                        continue;
1779                    }
1780                } else {
1781                    // Fire-and-forget: label operations are cosmetic (routing already
1782                    // persisted in DB). Spawning them prevents blocking the tick loop
1783                    // on 6-9 sequential HTTP calls (~3-9s overhead per routed task).
1784                    let backend_clone = Arc::clone(backend);
1785                    let task_id_clone = task.id.clone();
1786                    let task_labels: Vec<String> = task.labels.clone();
1787                    let new_labels = {
1788                        let mut labels = vec![
1789                            format!("agent:{}", result.agent),
1790                            format!("complexity:{}", result.complexity),
1791                        ];
1792                        if let Some(ref model) = result.model {
1793                            labels.push(format!("model:{model}"));
1794                        }
1795                        labels
1796                    };
1797                    tokio::spawn(async move {
1798                        // Remove old agent/complexity/model labels
1799                        for label in &task_labels {
1800                            if (label.starts_with("agent:")
1801                                || label.starts_with("complexity:")
1802                                || label.starts_with("model:"))
1803                                && let Err(e) =
1804                                    backend_clone.remove_label(&task_id_clone, label).await
1805                            {
1806                                tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1807                            }
1808                        }
1809                        // Add new labels
1810                        if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1811                        {
1812                            tracing::warn!(
1813                                task_id = task_id_clone.0,
1814                                ?e,
1815                                "failed to set routing labels"
1816                            );
1817                        }
1818                    });
1819
1820                    // Transition to routed
1821                    if let Err(e) = task_manager
1822                        .update_task_status(&task.id, Status::Routed)
1823                        .await
1824                    {
1825                        tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1826                        continue;
1827                    }
1828                }
1829
1830                // Dual-write: upsert task + store route result in SQLite
1831                match store.ensure_external_task(repo, task).await {
1832                    Ok(store_id) => {
1833                        let profile_json =
1834                            serde_json::to_string(&result.profile).unwrap_or_default();
1835                        if let Err(e) = store
1836                            .store_route(&crate::store::StoreRoute {
1837                                id: store_id,
1838                                agent: &result.agent,
1839                                model: result.model.as_deref(),
1840                                complexity: &result.complexity,
1841                                estimate: result.estimate,
1842                                reason: &result.reason,
1843                                profile: &profile_json,
1844                            })
1845                            .await
1846                        {
1847                            tracing::debug!(
1848                                task_id = task.id.0,
1849                                ?e,
1850                                "dual-write: store_route failed"
1851                            );
1852                        }
1853                        if let Err(e) = store.update_status(store_id, TaskStatus::Routed).await {
1854                            tracing::debug!(
1855                                task_id = task.id.0,
1856                                ?e,
1857                                "dual-write: update_status failed"
1858                            );
1859                        }
1860                        // Sync the estimate to GitHub Projects if the router assigned one.
1861                        // Fire-and-forget: do NOT await inline — GraphQL mutations can take
1862                        // up to 30 s and would block the entire tick loop on every routed task.
1863                        if result.estimate > 0 {
1864                            let backend_clone = Arc::clone(backend);
1865                            let task_id = task.id.clone();
1866                            let estimate = result.estimate;
1867                            tokio::spawn(async move {
1868                                if let Err(e) = backend_clone
1869                                    .sync_estimate_to_project(&task_id, estimate)
1870                                    .await
1871                                {
1872                                    tracing::debug!(
1873                                        task_id = task_id.0,
1874                                        estimate,
1875                                        err = %e,
1876                                        "dual-write: estimate project sync failed"
1877                                    );
1878                                }
1879                            });
1880                        }
1881                    }
1882                    Err(e) => {
1883                        tracing::debug!(
1884                            task_id = task.id.0,
1885                            ?e,
1886                            "dual-write: ensure_external_task failed"
1887                        );
1888                    }
1889                }
1890
1891                if let Some(ref warning) = result.warning {
1892                    tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1893                }
1894
1895                tracing::info!(
1896                    task_id = task.id.0,
1897                    agent = %result.agent,
1898                    complexity = %result.complexity,
1899                    reason = %result.reason,
1900                    "task routed"
1901                );
1902            }
1903            Err(e) => {
1904                if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1905                    tracing::error!(
1906                        task_id = %task.id.0,
1907                        scope = %cooled.scope(),
1908                        remaining_secs = ?cooled.remaining_secs(),
1909                        degraded = ?cooled.degraded(),
1910                        "ALL AGENTS COOLED — no agent available to route this task. \
1911                         Task stays in 'new' and will be retried on every tick. \
1912                         Check `orch cooldown list` to see what is blocking dispatch."
1913                    );
1914                    continue;
1915                }
1916                if let Some(waiting) = e.downcast_ref::<WaitingOnDependency>() {
1917                    tracing::info!(
1918                        task_id = %task.id.0,
1919                        blockers = %waiting.blockers.join(", "),
1920                        "task is waiting on a dependency"
1921                    );
1922                    continue;
1923                }
1924                tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1925            }
1926        }
1927    }
1928    Ok(())
1929}
1930
1931/// Global routing sweep: route status:new tasks from inactive or removed repos.
1932///
1933/// `tick_route_tasks` is scoped to the active repo per-tick and never processes
1934/// tasks whose `repo` field differs from the current project. When a project is
1935/// removed from the orch config, its tasks accumulate in `new` indefinitely
1936/// because no tick ever runs for that repo. This sweep covers all repos so
1937/// orphaned new tasks are eventually routed.
1938///
1939/// `active_repos` must be the full set of currently-configured project repos,
1940/// not just the repo of the tick that called this — mirroring
1941/// `dispatch_routed_tasks_global` (see its doc comment for the "racing the real
1942/// per-repo dispatch" failure mode this guards against). Filtering on
1943/// `t.repo != current_repo` alone (the previous behavior) made every *other*
1944/// active repo's new tasks look "orphaned" too and routed them a tick early,
1945/// through the wrong repo's tick.
1946pub(crate) async fn route_new_tasks_global(
1947    router: &mut Router,
1948    store: &Arc<TaskStore>,
1949    active_repos: &HashSet<String>,
1950    routing_quota: &mut RoutingTickQuota,
1951) -> anyhow::Result<()> {
1952    if routing_quota.is_exhausted() {
1953        tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1954        return Ok(());
1955    }
1956
1957    let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1958    let orphaned: Vec<&Task> = all_new
1959        .iter()
1960        .filter(|t| !active_repos.contains(&t.repo))
1961        .filter(|t| !skips_agent(&t.labels))
1962        .collect();
1963
1964    if orphaned.is_empty() {
1965        return Ok(());
1966    }
1967
1968    tracing::info!(
1969        count = orphaned.len(),
1970        "global routing sweep: found new tasks from inactive repos"
1971    );
1972
1973    router.refresh_health(store).await;
1974
1975    for task in orphaned.into_iter().take(routing_quota.remaining()) {
1976        if !routing_quota.consume_one() {
1977            break;
1978        }
1979        let ext_task = tasks::store_task_to_external(task);
1980        let task_start = Instant::now();
1981        match router.route(&ext_task, store, &task.repo).await {
1982            Ok(result) => {
1983                tracing::debug!(
1984                    task_id = %ext_task.id.0,
1985                    repo = %task.repo,
1986                    duration_ms = task_start.elapsed().as_millis(),
1987                    "global sweep: route completed"
1988                );
1989                let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1990                if let Err(e) = store
1991                    .store_route(&StoreRoute {
1992                        id: task.id,
1993                        agent: &result.agent,
1994                        model: result.model.as_deref(),
1995                        complexity: &result.complexity,
1996                        estimate: result.estimate,
1997                        reason: &result.reason,
1998                        profile: &profile_json,
1999                    })
2000                    .await
2001                {
2002                    tracing::warn!(
2003                        task_id = %ext_task.id.0,
2004                        repo = %task.repo,
2005                        err = %e,
2006                        "global sweep: failed to store route result; skipping status update"
2007                    );
2008                    continue;
2009                }
2010                match store.update_status(task.id, TaskStatus::Routed).await {
2011                    Err(e) => {
2012                        tracing::warn!(
2013                            task_id = %ext_task.id.0,
2014                            repo = %task.repo,
2015                            err = %e,
2016                            "global sweep: failed to set status:routed"
2017                        );
2018                    }
2019                    _ => {
2020                        tracing::info!(
2021                            task_id = %ext_task.id.0,
2022                            repo = %task.repo,
2023                            agent = %result.agent,
2024                            complexity = %result.complexity,
2025                            "global sweep: task routed"
2026                        );
2027                    }
2028                }
2029                if let Some(ref warning) = result.warning {
2030                    tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
2031                }
2032            }
2033            Err(e) => {
2034                if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
2035                    tracing::error!(
2036                        task_id = %ext_task.id.0,
2037                        repo = %task.repo,
2038                        scope = %cooled.scope(),
2039                        remaining_secs = ?cooled.remaining_secs(),
2040                        degraded = ?cooled.degraded(),
2041                        "global sweep: ALL AGENTS COOLED — task stays in 'new'"
2042                    );
2043                    continue;
2044                }
2045                if let Some(waiting) = e.downcast_ref::<WaitingOnDependency>() {
2046                    tracing::info!(
2047                        task_id = %ext_task.id.0,
2048                        repo = %task.repo,
2049                        blockers = %waiting.blockers.join(", "),
2050                        "global sweep: task is waiting on a dependency"
2051                    );
2052                    continue;
2053                }
2054                tracing::error!(
2055                    task_id = %ext_task.id.0,
2056                    repo = %task.repo,
2057                    err = ?e,
2058                    "global sweep: routing failed, skipping task"
2059                );
2060            }
2061        }
2062    }
2063
2064    Ok(())
2065}
2066
2067/// Phase 3b of tick: spawn agents for all status:routed tasks up to the parallel limit.
2068#[derive(Debug, Clone, Copy)]
2069pub(crate) struct DispatchMode {
2070    pub(crate) is_degraded: bool,
2071    pub(crate) healthy_agents: usize,
2072    pub(crate) threshold: usize,
2073}
2074
2075pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
2076    let threshold = router_config::min_healthy_agents_threshold();
2077    let healthy_agents = router.healthy_agent_count("simple");
2078    let is_degraded = router.is_degraded(threshold);
2079    DispatchMode {
2080        is_degraded,
2081        healthy_agents,
2082        threshold,
2083    }
2084}
2085
2086/// Phase 3b of tick: spawn agents for all status:routed tasks up to the parallel limit.
2087#[expect(
2088    clippy::too_many_arguments,
2089    reason = "each argument is engine state the tick passes down"
2090)]
2091#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
2092pub(crate) async fn tick_dispatch_tasks(
2093    backend: &Arc<dyn ExternalBackend>,
2094    tmux: &Arc<TmuxManager>,
2095    repo: &str,
2096    runner: &Arc<TaskRunner>,
2097    capture: &Arc<CaptureService>,
2098    semaphore: &Arc<Semaphore>,
2099    task_manager: &Arc<TaskManager>,
2100    weight_tx: &mpsc::Sender<WeightSignal>,
2101    dispatch_mode: DispatchMode,
2102    dispatching: &Arc<DashMap<String, String>>,
2103    store: &Arc<TaskStore>,
2104    session_map: &HashMap<String, bool>,
2105) -> anyhow::Result<()> {
2106    // Phase 3a already drops no-agent and owner:human, this filter stays as defense in depth
2107    let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
2108
2109    // Also include internal tasks in Routed status.
2110    let internal_routed = task_manager
2111        .list_internal_by_status(DbStatus::Routed)
2112        .await?;
2113    routed_tasks.extend(internal_routed);
2114
2115    // Filter and collect owned tasks to avoid lifetime issues
2116    let dispatchable: Vec<ExternalTask> = routed_tasks
2117        .into_iter()
2118        .filter(|t| !skips_agent(&t.labels))
2119        .collect();
2120
2121    dispatch_tasks_for_repo(
2122        backend,
2123        tmux,
2124        repo,
2125        runner,
2126        capture,
2127        semaphore,
2128        task_manager,
2129        weight_tx,
2130        dispatch_mode,
2131        dispatching,
2132        store,
2133        session_map,
2134        dispatchable,
2135    )
2136    .await
2137}
2138
2139/// Dispatch a pre-fetched list of routed tasks for a single repo.
2140///
2141/// Shared by `tick_dispatch_tasks` (active-repo, per-tick dispatch) and
2142/// `dispatch_routed_tasks_global` (inactive-repo sweep) so both paths spawn
2143/// agents through the exact same worktree/tmux/runner logic.
2144#[expect(
2145    clippy::too_many_arguments,
2146    reason = "each argument is engine state the tick passes down"
2147)]
2148async fn dispatch_tasks_for_repo(
2149    backend: &Arc<dyn ExternalBackend>,
2150    tmux: &Arc<TmuxManager>,
2151    repo: &str,
2152    runner: &Arc<TaskRunner>,
2153    capture: &Arc<CaptureService>,
2154    semaphore: &Arc<Semaphore>,
2155    task_manager: &Arc<TaskManager>,
2156    weight_tx: &mpsc::Sender<WeightSignal>,
2157    dispatch_mode: DispatchMode,
2158    dispatching: &Arc<DashMap<String, String>>,
2159    store: &Arc<TaskStore>,
2160    session_map: &HashMap<String, bool>,
2161    dispatchable: Vec<ExternalTask>,
2162) -> anyhow::Result<()> {
2163    if dispatchable.is_empty() {
2164        tracing::debug!(count = 0, "dispatchable tasks found");
2165        return Ok(());
2166    }
2167
2168    tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2169    if dispatch_mode.is_degraded {
2170        tracing::warn!(
2171            healthy_agents = dispatch_mode.healthy_agents,
2172            threshold = dispatch_mode.threshold,
2173            "degraded mode: using sequential dispatch"
2174        );
2175    }
2176
2177    let sequential_delay = if dispatch_mode.is_degraded {
2178        router_config::sequential_dispatch_delay_ms()
2179    } else {
2180        0
2181    };
2182
2183    let mut ranked: Vec<(Priority, ExternalTask)> = Vec::with_capacity(dispatchable.len());
2184    for task in dispatchable {
2185        let has_pr = match store.resolve_id_by_external(repo, &task.id.0).await {
2186            Ok(Some(id)) => store.get(id).await.is_ok_and(|t| t.pr_number.is_some()),
2187            _ => false,
2188        };
2189        ranked.push((Priority::from_labels(&task.labels).with_pr(has_pr), task));
2190    }
2191    priority::sort_by_priority(&mut ranked, |t| t.0);
2192
2193    for (idx, (priority, task)) in ranked.into_iter().enumerate() {
2194        if dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2195            continue;
2196        }
2197        // In degraded mode, add delay between dispatches to pace the system
2198        if idx > 0 && dispatch_mode.is_degraded {
2199            let delay_ms = sequential_delay;
2200            tracing::debug!(
2201                task_id = task.id.0,
2202                delay_ms = delay_ms,
2203                "sequential dispatch: waiting before dispatch"
2204            );
2205            time::sleep(Duration::from_millis(delay_ms)).await;
2206        }
2207        // In-memory guard: prevents double-dispatch due to GitHub API eventual consistency.
2208        // After update_status(InProgress), the label removal fires a webhook that can
2209        // trigger an immediate tick. GitHub's search index may not yet reflect the label
2210        // change, so list_by_status(Routed) can still return this task. The tmux session
2211        // does not exist until the runner completes worktree setup (~10s later), so the
2212        // session_exists check alone is insufficient.
2213        let dispatch_key = format!("{}/{}", repo, task.id.0);
2214        // Atomically claim the dispatch slot.  DashMap lets us distinguish:
2215        //   - same task already in-flight (expected): log debug and skip.
2216        //   - different task holds the same key (should never happen): log warning and skip.
2217        // A plain DashSet::insert would return false for both cases without distinction.
2218        {
2219            match dispatching.entry(dispatch_key.clone()) {
2220                Entry::Occupied(existing) => {
2221                    let existing_id = existing.get().clone();
2222                    drop(existing); // release shard lock before logging
2223                    if existing_id == task.id.0 {
2224                        tracing::debug!(
2225                            task_id = task.id.0,
2226                            "task already dispatching, skipping duplicate"
2227                        );
2228                    } else {
2229                        tracing::warn!(
2230                            task_id = task.id.0,
2231                            existing_task_id = existing_id,
2232                            dispatch_key,
2233                            "dispatch key collision: unexpected task already holds this key"
2234                        );
2235                    }
2236                    continue;
2237                }
2238                Entry::Vacant(slot) => {
2239                    slot.insert(task.id.0.clone());
2240                    // Promoted from debug! to info! (issue #3526): paired with the
2241                    // "dispatch guard released" line in dispatch_guard.rs, this is the
2242                    // only direct evidence of how long a dispatch guard was actually held
2243                    // for a given key. Both were debug!, silently dropped under the
2244                    // default "orch=info" filter (main.rs) — see the reclaim-skip log
2245                    // promotions above for the full rationale.
2246                    tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2247                }
2248            }
2249        }
2250        // RAII guard — removes dispatch_key on drop even if the spawned task panics.
2251        let dispatch_guard = DispatchGuard::new(Arc::clone(dispatching), dispatch_key.clone());
2252        let task_id = task.id.0.clone();
2253
2254        // Load routing result from store (stored during Phase 3a). Loaded before the
2255        // tmux/semaphore/in_progress steps so a stale cooldown can bail out cheaply.
2256        let route_result = match get_route_result(store, repo, &task_id).await {
2257            Ok(r) => Some(r),
2258            Err(RouteResultError::NoAgent { .. }) => {
2259                tracing::warn!(
2260                    task_id,
2261                    "routed task missing agent — resetting to new for re-routing (#1604)"
2262                );
2263                if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2264                    tracing::error!(task_id, error = %e2, "failed to reset task to new");
2265                }
2266                continue; // dispatch_guard drops here, removing the key
2267            }
2268            Err(e) => {
2269                tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2270                None
2271            }
2272        };
2273
2274        // Re-check the assigned agent/model against the cooldown store: routing and
2275        // dispatch are decoupled by ticks, and a failure recorded after this task was
2276        // routed but before it dispatched would otherwise burn a full run on an
2277        // already-cooled agent/model. Leave the task in Routed so the next tick's
2278        // dispatch pass re-evaluates it — no new per-task defer state (see #3243).
2279        if let Some(route) = route_result.as_ref() {
2280            let agent_cooled = is_agent_in_cooldown(&route.agent);
2281            let model_cooled = route
2282                .model
2283                .as_deref()
2284                .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2285            if agent_cooled || model_cooled {
2286                tracing::warn!(
2287                    task_id,
2288                    agent = route.agent,
2289                    model = route.model.as_deref().unwrap_or(""),
2290                    "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2291                );
2292                continue; // dispatch_guard drops here, removing the key
2293            }
2294        }
2295
2296        // Check if already running (has active session)
2297        let session_name = tmux.session_name(repo, &task_id);
2298        if tmux
2299            .session_blocks_dispatch_from_map(&session_name, session_map)
2300            .await
2301        {
2302            tracing::warn!(
2303                task_id,
2304                session_name,
2305                "task has existing tmux session, skipping dispatch"
2306            );
2307            continue; // dispatch_guard drops here, removing the key
2308        }
2309
2310        // Try to acquire a slot
2311        let Some(permit) = priority::acquire_dispatch_slot(semaphore, priority) else {
2312            tracing::debug!("all parallel slots busy, skipping remaining tasks");
2313            break; // dispatch_guard drops here, removing the key
2314        };
2315
2316        // Mark in_progress BEFORE spawning to prevent double dispatch.
2317        let set_in_progress_result = task_manager
2318            .update_task_status(&task.id, Status::InProgress)
2319            .await;
2320        if let Err(e) = set_in_progress_result {
2321            tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2322            drop(permit);
2323            continue; // dispatch_guard drops here, removing the key
2324        }
2325        tracing::info!(task_id, "dispatching task");
2326
2327        // Register session for capture
2328        capture
2329            .register_session(repo, &task_id, &session_name)
2330            .await;
2331
2332        // Dispatch task
2333        let runner = Arc::clone(runner);
2334        let backend = Arc::clone(backend);
2335        let tmux = Arc::clone(tmux);
2336        let capture = Arc::clone(capture);
2337        let task_id_for_cleanup = task_id.clone();
2338        let task_owned = task;
2339        let weight_tx = weight_tx.clone();
2340        let repo_owned = repo.to_string();
2341        let task_manager_for_spawn = Arc::clone(task_manager);
2342        let store_for_spawn = Arc::clone(store);
2343
2344        let repo_ctx = repo_owned.clone();
2345        tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2346            let _dispatch_guard = dispatch_guard; // released on drop (normal or panic)
2347            let dispatch_start = Instant::now();
2348            match runner
2349                .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2350                .await
2351            {
2352                Ok(signal) => {
2353                    tracing::info!(task_id, "task runner completed");
2354
2355                    let duration = dispatch_start.elapsed().as_secs_f64();
2356
2357                    // Derive a display status from the weight signal.
2358                    // When review agent is enabled, successful tasks go through
2359                    // review before being marked done — UNLESS the response
2360                    // handler already marked the task done (no-op tasks with
2361                    // no PR and no commits should not enter the review cycle).
2362                    let enable_review = config::get("workflow.enable_review_agent")
2363                        .map_or(true, |v| v != "false");
2364                    // Only a successful run can reach the no-PR shortcut, so only then read the PR.
2365                    let has_pr = if enable_review && matches!(signal, WeightSignal::Success { .. })
2366                    {
2367                        match store_for_spawn.resolve_task_id(&repo_owned, &task_id).await {
2368                            Ok(Some(store_id)) => match store_for_spawn.get(store_id).await {
2369                                Ok(t) => t.pr_number.is_some(),
2370                                Err(e) => {
2371                                    tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2372                                    true
2373                                }
2374                            },
2375                            Ok(None) => false,
2376                            Err(e) => {
2377                                tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2378                                true
2379                            }
2380                        }
2381                    } else {
2382                        false
2383                    };
2384                    let display_status = display_status_for(&signal, enable_review, has_pr);
2385
2386                    if display_status == "done" {
2387                        tracing::info!(
2388                            task_id,
2389                            "marking done: the run succeeded with no PR to review"
2390                        );
2391                    }
2392                    tracing::info!(task_id, display_status, "task run finished, applying status");
2393
2394                    // Send weight signal back to the router
2395                    let _ = weight_tx.send(signal).await;
2396
2397                    // All tasks: if needs_review, trigger the review agent.
2398                    // Status updates go through task_manager so internal tasks
2399                    // hit SQLite while external tasks hit GitHub labels.
2400                    if display_status == "needs_review" {
2401                        // Transition to NeedsReview — emits event so the review subscriber
2402                        // fires with the correct duration before the review agent starts.
2403                        if let Err(e) = task_manager_for_spawn
2404                            .update_task_status_with_duration(
2405                                &ExternalId(task_id.clone()),
2406                                Status::NeedsReview,
2407                                Some(duration),
2408                            )
2409                            .await
2410                        {
2411                            tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2412                        }
2413                        // Review agent dispatch is handled by the event-driven subscriber.
2414                    } else {
2415                        // done, blocked, or new (rate-limited): update status directly.
2416                        let final_status = match display_status {
2417                            "done" => Status::Done,
2418                            "blocked" => Status::Blocked,
2419                            _ => Status::New,
2420                        };
2421                        if let Err(e) = task_manager_for_spawn
2422                            .update_task_status_with_duration(
2423                                &ExternalId(task_id.clone()),
2424                                final_status,
2425                                Some(duration),
2426                            )
2427                            .await
2428                        {
2429                            tracing::warn!(task_id, ?e, "failed to update task status after completion");
2430                        }
2431                    }
2432                }
2433                Err(e) => {
2434                    tracing::error!(task_id, ?e, "task runner failed");
2435                    let duration = dispatch_start.elapsed().as_secs_f64();
2436                    if e.downcast_ref::<NoProjectDir>().is_some() {
2437                        // The run failed before any agent started, so nothing exists to review
2438                        block_task_without_project(
2439                            &task_manager_for_spawn,
2440                            &store_for_spawn,
2441                            &repo_owned,
2442                            &task_id,
2443                        )
2444                        .await;
2445                    } else if is_internal_id(&task_id) {
2446                        // Internal tasks have no GitHub issue to comment on.
2447                        if let Err(ue) = task_manager_for_spawn
2448                            .update_task_status_with_duration(
2449                                &ExternalId(task_id.clone()),
2450                                Status::NeedsReview,
2451                                Some(duration),
2452                            )
2453                            .await
2454                        {
2455                            tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2456                        }
2457                    } else {
2458                        // Post comment (best-effort)
2459                        if let Err(comment_err) = backend
2460                            .post_comment(
2461                                &ExternalId(task_id.clone()),
2462                                &format!(
2463                                    "[{}] error: task runner failed: {e}{}",
2464                                    chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2465                                    engine::orch_footer()
2466                                ),
2467                            )
2468                            .await
2469                        {
2470                            tracing::warn!(
2471                                task_id,
2472                                ?comment_err,
2473                                "failed to post error comment to GitHub"
2474                            );
2475                        }
2476                        // Update status immediately to NeedsReview so external tasks
2477                        // don't remain stuck in InProgress until the no-session timeout.
2478                        // The notify subscriber will send the channel notification with duration.
2479                        if let Err(ue) = task_manager_for_spawn
2480                            .update_task_status_with_duration(
2481                                &ExternalId(task_id.clone()),
2482                                Status::NeedsReview,
2483                                Some(duration),
2484                            )
2485                            .await
2486                        {
2487                            tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2488                        }
2489                    }
2490                }
2491            }
2492
2493            // Unregister session from capture
2494            capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2495
2496            // Release the semaphore permit
2497            drop(permit);
2498            // _dispatch_guard drops here, removing the key from the dispatching set.
2499        }));
2500    }
2501    Ok(())
2502}
2503
2504/// Block a task whose repo has no project directory. Nothing exists to review, so
2505/// `NeedsReview` would only feed the refire and rebroadcast loop (#4024)
2506async fn block_task_without_project(
2507    task_manager: &Arc<TaskManager>,
2508    store: &Arc<TaskStore>,
2509    repo: &str,
2510    task_id: &str,
2511) {
2512    let store_id = match store.resolve_task_id(repo, task_id).await {
2513        Ok(Some(id)) => id,
2514        Ok(None) => {
2515            tracing::error!(
2516                task_id,
2517                repo,
2518                "cannot block task without project: not in the store"
2519            );
2520            return;
2521        }
2522        Err(e) => {
2523            tracing::error!(task_id, repo, err = %e, "cannot block task without project: store lookup failed");
2524            return;
2525        }
2526    };
2527    let fields = [
2528        (
2529            "block_reason",
2530            serde_json::json!(NoProjectDir::BLOCK_REASON),
2531        ),
2532        (
2533            "last_error",
2534            serde_json::json!(format!("no project directory for repo {repo}")),
2535        ),
2536    ];
2537    if let Err(e) = task_manager
2538        .update_task_status_and_result_by_store_id(
2539            store_id,
2540            repo,
2541            &ExternalId(task_id.to_string()),
2542            Status::Blocked,
2543            &fields,
2544        )
2545        .await
2546    {
2547        tracing::error!(task_id, repo, err = %e, "failed to block task without project");
2548    }
2549}
2550
2551fn skips_agent(labels: &[String]) -> bool {
2552    labels
2553        .iter()
2554        .any(|label| label == "no-agent" || label == "owner:human")
2555}
2556
2557/// Filter routed tasks down to those belonging to a repo outside `active_repos`,
2558/// drop `no-agent` and `owner:human` tasks, and group the rest by repo.
2559///
2560/// Pure/side-effect-free so it can be unit tested without touching real tmux
2561/// sessions or spawning a runner — see `dispatch_routed_tasks_global`, which is
2562/// the only caller and does the actual (side-effectful) dispatch.
2563fn group_orphaned_routed_tasks(
2564    all_routed: Vec<Task>,
2565    active_repos: &HashSet<String>,
2566) -> HashMap<String, Vec<Task>> {
2567    let mut by_repo: HashMap<String, Vec<Task>> = HashMap::new();
2568    for task in all_routed
2569        .into_iter()
2570        .filter(|t| !active_repos.contains(&t.repo))
2571        .filter(|t| !skips_agent(&t.labels))
2572    {
2573        by_repo.entry(task.repo.clone()).or_default().push(task);
2574    }
2575    by_repo
2576}
2577
2578/// Global dispatch sweep: dispatch status:routed tasks from inactive or removed repos.
2579///
2580/// `tick_dispatch_tasks` is scoped to the active repo per-tick (its `task_manager`
2581/// only queries that repo's tasks) and never processes tasks whose `repo` field
2582/// differs from the current project. When a project is removed from the orch
2583/// config, tasks that reach `routed` for it — including ones routed by the
2584/// `route_new_tasks_global` sweep — have no code path left to dispatch them, so
2585/// they stay `routed` forever. This sweep covers all repos so orphaned routed
2586/// tasks are eventually dispatched.
2587///
2588/// `active_repos` must be the full set of currently-configured project repos,
2589/// not just one project's repo; passing only that project's own repo would make
2590/// every other active repo's routed tasks look "orphaned" too and dispatch them
2591/// a second time through this fallback path, racing the real per-repo dispatch.
2592///
2593/// `TaskRunner::resolve_project_dir` already falls back to the deterministic
2594/// `~/.orch/projects/<owner>/<repo>.git` bare-clone convention when a repo is
2595/// not listed in the active config, so dispatch works without a `ProjectEngine`.
2596#[expect(
2597    clippy::too_many_arguments,
2598    reason = "each argument is engine state the tick passes down"
2599)]
2600pub(crate) async fn dispatch_routed_tasks_global(
2601    tmux: &Arc<TmuxManager>,
2602    capture: &Arc<CaptureService>,
2603    semaphore: &Arc<Semaphore>,
2604    weight_tx: &mpsc::Sender<WeightSignal>,
2605    dispatch_mode: DispatchMode,
2606    dispatching: &Arc<DashMap<String, String>>,
2607    store: &Arc<TaskStore>,
2608    session_map: &HashMap<String, bool>,
2609    active_repos: &HashSet<String>,
2610) -> anyhow::Result<()> {
2611    let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2612    let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2613
2614    if by_repo.is_empty() {
2615        return Ok(());
2616    }
2617
2618    tracing::info!(
2619        repos = by_repo.len(),
2620        "global dispatch sweep: found routed tasks from inactive repos"
2621    );
2622
2623    for (repo, tasks) in by_repo {
2624        let backend: Arc<dyn ExternalBackend> = match GitHubBackend::new(repo.clone()) {
2625            Ok(b) => Arc::new(b),
2626            Err(e) => {
2627                tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2628                continue;
2629            }
2630        };
2631        let task_manager = Arc::new(TaskManager::with_store(
2632            Arc::clone(&backend),
2633            Arc::clone(store),
2634            repo.clone(),
2635        ));
2636        let runner = Arc::new(TaskRunner::new(repo.clone()).with_store(Arc::clone(store)));
2637
2638        // Without a project directory the runner bails before any agent starts, so
2639        // block now instead of taking a dispatch slot first
2640        if let Err(e) = runner.resolve_project_dir().await
2641            && e.downcast_ref::<NoProjectDir>().is_some()
2642        {
2643            tracing::warn!(repo = %repo, count = tasks.len(), "global sweep: repo has no project directory, blocking its routed tasks");
2644            for task in &tasks {
2645                let ext = task
2646                    .external_id
2647                    .clone()
2648                    .unwrap_or_else(|| format!("internal:{}", task.id));
2649                block_task_without_project(&task_manager, store, &repo, &ext).await;
2650            }
2651            continue;
2652        }
2653
2654        let dispatchable: Vec<ExternalTask> =
2655            tasks.iter().map(tasks::store_task_to_external).collect();
2656
2657        if let Err(e) = dispatch_tasks_for_repo(
2658            &backend,
2659            tmux,
2660            &repo,
2661            &runner,
2662            capture,
2663            semaphore,
2664            &task_manager,
2665            weight_tx,
2666            dispatch_mode,
2667            dispatching,
2668            store,
2669            session_map,
2670            dispatchable,
2671        )
2672        .await
2673        {
2674            tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2675        }
2676    }
2677
2678    Ok(())
2679}
2680
2681/// Phase 4 of tick: unblock parent tasks whose sub-issues are all done.
2682pub(crate) async fn tick_unblock_parents(
2683    backend: &Arc<dyn ExternalBackend>,
2684    task_manager: &Arc<TaskManager>,
2685    store: &Arc<TaskStore>,
2686    repo: &str,
2687) -> anyhow::Result<()> {
2688    let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2689
2690    // Pre-filter: skip tasks that are blocked for a known reason (e.g. CI failure,
2691    // escalation). These are not waiting on children, so there is no point making
2692    // a GraphQL call for them.
2693    let store_blocked = store.list_by_status(repo, TaskStatus::Blocked).await?;
2694    let has_block_reason: HashSet<String> = store_blocked
2695        .iter()
2696        .filter(|t| t.block_reason.is_some())
2697        .filter_map(|t| t.external_id.clone())
2698        .collect();
2699
2700    // Spawn a concurrent task for each candidate blocked task.
2701    // Previously these ran sequentially; get_sub_issues is a GraphQL call taking 1-3 s each,
2702    // and with 4 blocked tasks that was 4-12 s of sequential latency every tick.
2703    let handles: Vec<_> = blocked
2704        .into_iter()
2705        .filter(|task| {
2706            if has_block_reason.contains(&task.id.0) {
2707                tracing::debug!(
2708                    task_id = task.id.0,
2709                    "skipping blocked task with known block_reason"
2710                );
2711                false
2712            } else {
2713                true
2714            }
2715        })
2716        .map(|task| {
2717            let backend_clone = Arc::clone(backend);
2718            let task_manager_clone = Arc::clone(task_manager);
2719            let store_clone = Arc::clone(store);
2720            let repo_owned = repo.to_string();
2721            tokio::spawn(async move {
2722                let task_id = &task.id;
2723
2724                let mut children: Vec<ExternalId> = Vec::new();
2725                let mut child_statuses: HashMap<
2726                    String,
2727                    TaskStatus,
2728                > = HashMap::new();
2729
2730                match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2731                    Ok(Some(parent_store_id)) => {
2732                        match store_clone.list_children(&repo_owned, parent_store_id).await {
2733                            Ok(store_children) => {
2734                                for child in store_children {
2735                                    let Some(child_external_id) = child.external_id else {
2736                                        continue;
2737                                    };
2738                                    child_statuses.insert(child_external_id.clone(), child.status);
2739                                    children.push(ExternalId(child_external_id));
2740                                }
2741                            }
2742                            Err(e) => {
2743                                tracing::debug!(
2744                                    task_id = task_id.0,
2745                                    ?e,
2746                                    "failed to list store-linked children"
2747                                );
2748                            }
2749                        }
2750                    }
2751                    Ok(None) => {}
2752                    Err(e) => {
2753                        tracing::debug!(
2754                            task_id = task_id.0,
2755                            ?e,
2756                            "failed to resolve blocked task in store for child lookup"
2757                        );
2758                    }
2759                }
2760
2761                if !is_internal_id(&task_id.0) {
2762                    match backend_clone.get_sub_issues(task_id).await {
2763                        Ok(ids) => {
2764                            let mut seen: HashSet<String> =
2765                                children.iter().map(|child| child.0.clone()).collect();
2766                            for id in ids {
2767                                if seen.insert(id.0.clone()) {
2768                                    children.push(id);
2769                                }
2770                            }
2771                        }
2772                        Err(e) => {
2773                            tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2774                        }
2775                    }
2776                }
2777
2778                // No children means nothing to wait on — skip (may be blocked for other reasons)
2779                if children.is_empty() {
2780                    return;
2781                }
2782
2783                // Check if every child is done using the local store (batched) and
2784                // fall back to the backend only for children not present in the store.
2785                let mut all_done = true;
2786                // Build owned strings for the batch lookup (needed for &str slices below)
2787                let child_ext_strs: Vec<String> =
2788                    children.iter().map(|c| c.0.clone()).collect();
2789                let child_exts: Vec<&str> =
2790                    child_ext_strs.iter().map(String::as_str).collect();
2791                // Query store for statuses of any children present locally
2792                match store_clone
2793                    .get_statuses_by_external_ids(&repo_owned, &child_exts)
2794                    .await
2795                { Ok(map) => {
2796                    for (k, v) in map {
2797                        child_statuses.insert(k, v);
2798                    }
2799                } _ => {
2800                    tracing::debug!(
2801                        task_id = task_id.0,
2802                        "failed to batch-query store for child statuses; falling back to per-child lookups"
2803                    );
2804                }}
2805
2806                for child_id in &children {
2807                    // Check store first
2808                    if let Some(status) = child_statuses.get(&child_id.0) {
2809                        if *status != TaskStatus::Done {
2810                            all_done = false;
2811                            break;
2812                        }
2813                        continue;
2814                    }
2815
2816                    // Not found in store — fall back to backend check
2817                    match backend_clone.get_task(child_id).await {
2818                        Ok(child) => {
2819                            if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2820                                all_done = false;
2821                                break;
2822                            }
2823                        }
2824                        Err(e) => {
2825                            tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2826                            all_done = false;
2827                            break;
2828                        }
2829                    }
2830                }
2831
2832                if all_done {
2833                    tracing::info!(
2834                        task_id = task_id.0,
2835                        children = children.len(),
2836                        "all children done, unblocking parent"
2837                    );
2838                    if let Err(e) =
2839                        task_manager_clone.update_task_status(task_id, Status::New).await
2840                    {
2841                        tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2842                    }
2843                }
2844            })
2845        })
2846        .collect();
2847
2848    for handle in handles {
2849        if let Err(e) = handle.await {
2850            tracing::warn!(?e, "phase 4 sub-issue check panicked");
2851        }
2852    }
2853    Ok(())
2854}
2855
2856/// Phase 5 of tick: run cron job matching and fire any due jobs.
2857pub(crate) async fn tick_job_scheduler(
2858    jobs_path: &Path,
2859    backend: &Arc<dyn ExternalBackend>,
2860    store: Option<&Arc<TaskStore>>,
2861    repo: &str,
2862    transport: Option<&Arc<Transport>>,
2863) -> anyhow::Result<()> {
2864    jobs::tick(jobs_path, backend, store, repo, transport).await
2865}
2866
2867/// Core tick for one project, up to the routing phase.
2868///
2869/// Every phase here is scoped to one project, so all projects can run this at the same time:
2870/// 1. `tick_check_session_completions` — poll tmux for finished sessions
2871/// 2. `tick_detect_silent_agents`      — cooldown agents that produced no output
2872/// 3. `tick_recover_stuck_tasks`       — reset in_progress/in_review tasks with no session
2873///
2874/// The store-wide work that used to sit in here runs once per tick in `tick_all`:
2875/// `tick_global_sweeps` and `reconcile_dependencies`.
2876#[expect(
2877    clippy::too_many_arguments,
2878    reason = "each argument is engine state the tick passes down"
2879)]
2880pub(crate) async fn tick_before_routing(
2881    backend: &Arc<dyn ExternalBackend>,
2882    tmux: &Arc<TmuxManager>,
2883    repo: &str,
2884    capture: &Arc<CaptureService>,
2885    config: &EngineConfig,
2886    task_manager: &Arc<TaskManager>,
2887    dispatching: &Arc<DashMap<String, String>>,
2888    store: &Arc<TaskStore>,
2889    session_map: &HashMap<String, bool>,
2890) {
2891    let mut phases: Vec<(&str, u128)> = Vec::new();
2892    let mut at = Instant::now();
2893
2894    if let Err(e) = tick_check_session_completions(tmux, repo, capture, store).await {
2895        tracing::warn!(err = %e, "session completion phase failed");
2896    }
2897    phases.push(("sessions", at.elapsed().as_millis()));
2898
2899    // Every node tends its own sessions, only the leader decides. Followers dispatch once
2900    // tasks are assigned to nodes
2901    if !store.is_leader() {
2902        super::log_phase_timings("tick.pre_routing", repo, &phases);
2903        return;
2904    }
2905
2906    at = Instant::now();
2907    if let Err(e) =
2908        tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await
2909    {
2910        tracing::warn!(err = %e, "silent-agent detection phase failed");
2911    }
2912    phases.push(("silent_agents", at.elapsed().as_millis()));
2913
2914    at = Instant::now();
2915    if let Err(e) = tick_recover_stuck_tasks(
2916        backend,
2917        tmux,
2918        repo,
2919        task_manager,
2920        config,
2921        store,
2922        session_map,
2923        dispatching,
2924    )
2925    .await
2926    {
2927        tracing::warn!(err = %e, "stuck-task recovery phase failed");
2928    }
2929    phases.push(("stuck_tasks", at.elapsed().as_millis()));
2930
2931    super::log_phase_timings("tick.pre_routing", repo, &phases);
2932}
2933
2934/// Core tick for one project, after the routing phase.
2935///
2936/// Takes the router's decision as a `DispatchMode` value instead of the router itself, so
2937/// every project can run this at the same time:
2938/// 1. `tick_dispatch_tasks`  — spawn agents for status:routed tasks
2939/// 2. `tick_unblock_parents` — unblock parents whose sub-issues are all done
2940/// 3. `tick_job_scheduler`   — fire due cron jobs
2941#[expect(
2942    clippy::too_many_arguments,
2943    reason = "each argument is engine state the tick passes down"
2944)]
2945pub(crate) async fn tick_after_routing(
2946    backend: &Arc<dyn ExternalBackend>,
2947    tmux: &Arc<TmuxManager>,
2948    repo: &str,
2949    runner: &Arc<TaskRunner>,
2950    capture: &Arc<CaptureService>,
2951    semaphore: &Arc<Semaphore>,
2952    project_dir: &Path,
2953    task_manager: &Arc<TaskManager>,
2954    weight_tx: &mpsc::Sender<WeightSignal>,
2955    dispatch_mode: DispatchMode,
2956    dispatching: &Arc<DashMap<String, String>>,
2957    store: &Arc<TaskStore>,
2958    transport: Option<&Arc<Transport>>,
2959    session_map: &HashMap<String, bool>,
2960) {
2961    let mut phases: Vec<(&str, u128)> = Vec::new();
2962    let mut at = Instant::now();
2963
2964    if let Err(e) = tick_dispatch_tasks(
2965        backend,
2966        tmux,
2967        repo,
2968        runner,
2969        capture,
2970        semaphore,
2971        task_manager,
2972        weight_tx,
2973        dispatch_mode,
2974        dispatching,
2975        store,
2976        session_map,
2977    )
2978    .await
2979    {
2980        tracing::warn!(err = %e, "dispatch phase failed");
2981    }
2982    phases.push(("dispatch", at.elapsed().as_millis()));
2983
2984    at = Instant::now();
2985    if let Err(e) = tick_unblock_parents(backend, task_manager, store, repo).await {
2986        tracing::warn!(err = %e, "parent-unblock phase failed");
2987    }
2988    phases.push(("unblock_parents", at.elapsed().as_millis()));
2989
2990    at = Instant::now();
2991    let jobs_path = project_dir.join(".orch.yml");
2992    if let Err(e) = tick_job_scheduler(&jobs_path, backend, Some(store), repo, transport).await {
2993        tracing::error!(?e, "job scheduler tick failed");
2994    }
2995    phases.push(("jobs", at.elapsed().as_millis()));
2996
2997    super::log_phase_timings("tick.post_routing", repo, &phases);
2998}
2999
3000/// Core tick — runs every 10s for every project.
3001///
3002/// Projects no longer wait on each other: the phases around routing run for all of them at
3003/// once, so tick time tracks the slowest project instead of the sum. Routing stays serial
3004/// because it needs the router write lock and one shared `RoutingTickQuota`, which is what
3005/// gives `router.max_tasks_per_tick` its meaning.
3006///
3007/// The tmux session map, the global sweeps and the dependency reconcile are the same for
3008/// every project, so they run once per tick rather than once per project.
3009#[expect(
3010    clippy::too_many_arguments,
3011    reason = "each argument is engine state the tick passes down"
3012)]
3013pub(crate) async fn tick_all(
3014    engines: &[ProjectEngine],
3015    tmux: &Arc<TmuxManager>,
3016    capture: &Arc<CaptureService>,
3017    semaphore: &Arc<Semaphore>,
3018    config: &EngineConfig,
3019    router: &Arc<RwLock<Router>>,
3020    weight_tx: &mpsc::Sender<WeightSignal>,
3021    dispatching: &Arc<DashMap<String, String>>,
3022    transport: Option<&Arc<Transport>>,
3023    active_repos: &HashSet<String>,
3024    auto_merge_in_flight: &Arc<DashSet<String>>,
3025) {
3026    let _tick_span = tracing::info_span!("engine.tick").entered();
3027
3028    let Some(first) = engines.first() else {
3029        return;
3030    };
3031    // All project engines share one SQLite store, so leadership and the global sweeps below
3032    // are the same whichever project they are read from
3033    let store = &first.store;
3034
3035    // Calls made since the previous tick, so hourly quota exhaustion can be traced
3036    for (project, call_type, calls) in http::drain_call_counts().into_iter().take(10) {
3037        tracing::debug!(
3038            project,
3039            call_type,
3040            calls,
3041            "github api calls since last tick"
3042        );
3043    }
3044
3045    // One `tmux list-panes -a` for every project's stuck-task recovery, instead of one each
3046    let session_map = tmux.batch_session_active().await;
3047
3048    let mut passes = Vec::with_capacity(engines.len());
3049    for engine in engines {
3050        passes.push(REPO_CONTEXT.scope(
3051            engine.repo.clone(),
3052            tick_before_routing(
3053                &engine.backend,
3054                tmux,
3055                &engine.repo,
3056                capture,
3057                config,
3058                &engine.task_manager,
3059                dispatching,
3060                &engine.store,
3061                &session_map,
3062            ),
3063        ));
3064    }
3065    future::join_all(passes).await;
3066
3067    if !store.is_leader() {
3068        return;
3069    }
3070
3071    // Store-wide, so once per tick rather than once per project. Both read every repo, and
3072    // the NeedsReview sweep increments a counter it just read, which concurrent per-project
3073    // passes would each consume (#3864)
3074    let at = Instant::now();
3075    {
3076        let router_guard = router.read().await;
3077        tick_global_sweeps(
3078            &first.task_manager,
3079            store,
3080            &router_guard,
3081            dispatching,
3082            auto_merge_in_flight,
3083            active_repos,
3084        )
3085        .await;
3086    }
3087    let sweeps_ms = at.elapsed().as_millis();
3088
3089    let at = Instant::now();
3090    recover_untracked_repo_stuck_tasks(
3091        tmux,
3092        config,
3093        store,
3094        &session_map,
3095        dispatching,
3096        active_repos,
3097    )
3098    .await;
3099    let untracked_ms = at.elapsed().as_millis();
3100
3101    let at = Instant::now();
3102    if let Err(err) = dependencies::reconcile_dependencies(
3103        first.backend.as_ref(),
3104        store,
3105        transport.map(AsRef::as_ref),
3106    )
3107    .await
3108    {
3109        tracing::warn!(err = %err, "dependency reconcile failed");
3110    }
3111    super::log_phase_timings(
3112        "tick.global",
3113        &first.repo,
3114        &[
3115            ("global_sweeps", sweeps_ms),
3116            ("untracked_stuck", untracked_ms),
3117            ("dependencies", at.elapsed().as_millis()),
3118        ],
3119    );
3120
3121    {
3122        let mut router_guard = router.write().await;
3123        let mut routing_quota = RoutingTickQuota::new(router_config::max_tasks_per_routing_tick());
3124        for engine in engines {
3125            let at = Instant::now();
3126            REPO_CONTEXT
3127                .scope(engine.repo.clone(), async {
3128                    if let Err(e) = tick_route_tasks(
3129                        &engine.backend,
3130                        &engine.task_manager,
3131                        &mut router_guard,
3132                        &engine.store,
3133                        &engine.repo,
3134                        &mut routing_quota,
3135                    )
3136                    .await
3137                    {
3138                        tracing::warn!(err = %e, "routing phase failed");
3139                    }
3140                })
3141                .await;
3142            super::log_phase_timings(
3143                "tick.routing",
3144                &engine.repo,
3145                &[("route", at.elapsed().as_millis())],
3146            );
3147        }
3148
3149        // Global routing sweep for tasks from inactive or removed repos.
3150        // tick_route_tasks is scoped to the active repo; tasks from projects no longer
3151        // in config are never returned by list_routable and would stay in 'new' indefinitely.
3152        if let Err(e) =
3153            route_new_tasks_global(&mut router_guard, store, active_repos, &mut routing_quota).await
3154        {
3155            tracing::warn!(err = %e, "global new-task routing sweep failed");
3156        }
3157    }
3158
3159    let dispatch_mode = dispatch_mode_from_router(&*router.read().await);
3160
3161    // Re-read the session map: the completion phase above kills the sessions of finished
3162    // agents, and dispatch must not treat a killed session as one that still blocks a task
3163    let session_map = tmux.batch_session_active().await;
3164
3165    let mut passes = Vec::with_capacity(engines.len());
3166    for engine in engines {
3167        passes.push(REPO_CONTEXT.scope(
3168            engine.repo.clone(),
3169            tick_after_routing(
3170                &engine.backend,
3171                tmux,
3172                &engine.repo,
3173                &engine.runner,
3174                capture,
3175                semaphore,
3176                &engine.project_dir,
3177                &engine.task_manager,
3178                weight_tx,
3179                dispatch_mode,
3180                dispatching,
3181                &engine.store,
3182                transport,
3183                &session_map,
3184            ),
3185        ));
3186    }
3187    future::join_all(passes).await;
3188
3189    // Global dispatch sweep for routed tasks from inactive or removed repos.
3190    // tick_dispatch_tasks is scoped to each active repo; tasks from projects no longer
3191    // in config are never returned by list_external_by_status and would stay in
3192    // 'routed' indefinitely (see #3413). Excludes the full active-repo set, not just
3193    // one repo, so it doesn't treat other active repos as orphaned.
3194    if let Err(e) = dispatch_routed_tasks_global(
3195        tmux,
3196        capture,
3197        semaphore,
3198        weight_tx,
3199        dispatch_mode,
3200        dispatching,
3201        store,
3202        &session_map,
3203        active_repos,
3204    )
3205    .await
3206    {
3207        tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
3208    }
3209}
3210
3211/// Status a finished run moves to. Only a successful run with no PR (or with the review agent
3212/// off) becomes `done`. A failed run stays in `needs_review` with its issue open
3213pub(crate) fn display_status_for(
3214    signal: &WeightSignal,
3215    enable_review: bool,
3216    has_pr: bool,
3217) -> &'static str {
3218    match signal {
3219        WeightSignal::Success { .. } if enable_review => {
3220            if has_pr {
3221                "needs_review"
3222            } else {
3223                "done"
3224            }
3225        }
3226        // Silence detection or another non-rate-limit reset already moved the task to "new".
3227        // Converting it to needs_review would trigger the no-code shortcut and falsely close it
3228        WeightSignal::RateLimited { .. } | WeightSignal::Rerouted => "new",
3229        WeightSignal::Blocked => "blocked",
3230        // The review path re-routes a task with no PR and no commits
3231        WeightSignal::Failed { .. } => "needs_review",
3232        // None means the runner guard skipped the task or the status was unrecognised
3233        WeightSignal::None if enable_review => "needs_review",
3234        WeightSignal::Success { .. } | WeightSignal::None => "done",
3235    }
3236}
3237
3238#[cfg(test)]
3239mod tests {
3240    use super::*;
3241    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3242    use crate::channels::transport::Transport;
3243    use crate::engine::RunTimeouts;
3244    use crate::engine::cooldown;
3245    use crate::engine::router::Router;
3246    use crate::engine::router::RouterConfig;
3247    use crate::engine::tasks::TaskManager;
3248    use crate::store::DepSource;
3249    use crate::store::NewDependency;
3250    use crate::store::NewTask;
3251    use crate::store::StoreRoute;
3252    use crate::store::TaskStatus as DbStatus;
3253    use crate::store::TaskStatus;
3254    use crate::store::TaskStore;
3255    use crate::store::UpsertExternal;
3256    use crate::tmux::{SessionGuard, TmuxManager};
3257    use async_trait::async_trait;
3258    use dashmap::mapref::entry::Entry;
3259    use std::cell::Cell;
3260    use std::fs;
3261    use std::path::PathBuf;
3262    use std::sync::{Arc, Mutex};
3263    use tokio::sync::RwLock;
3264    use tokio::sync::oneshot;
3265    use tokio::task;
3266
3267    #[test]
3268    fn timed_out_run_without_pr_never_becomes_done() {
3269        let failed = WeightSignal::Failed {
3270            agent: "claude".to_string(),
3271        };
3272        assert_eq!(display_status_for(&failed, true, false), "needs_review");
3273        assert_eq!(display_status_for(&failed, false, false), "needs_review");
3274    }
3275
3276    #[test]
3277    fn done_needs_a_successful_run_with_no_pr() {
3278        let ok = WeightSignal::Success {
3279            agent: "claude".to_string(),
3280        };
3281        assert_eq!(display_status_for(&ok, true, true), "needs_review");
3282        assert_eq!(display_status_for(&ok, true, false), "done");
3283        assert_eq!(
3284            display_status_for(&WeightSignal::Rerouted, true, false),
3285            "new"
3286        );
3287        assert_eq!(
3288            display_status_for(&WeightSignal::None, true, false),
3289            "needs_review"
3290        );
3291    }
3292    #[test]
3293    fn skips_agent_for_human_and_no_agent_labels() {
3294        assert!(skips_agent(&["owner:human".to_string()]));
3295        assert!(skips_agent(&["no-agent".to_string()]));
3296        assert!(!skips_agent(&["status:new".to_string()]));
3297    }
3298
3299    /// A router reporting one available agent with no healthy "review" model, so the
3300    /// global NeedsReview refire sweep in `tick_global_sweeps` treats every review agent
3301    /// as cooled and skips — keeping it a no-op for tests unrelated to it.
3302    fn cooled_review_router_for_test() -> Router {
3303        Router::new_for_test(RouterConfig::default(), vec!["claude".to_string()])
3304    }
3305
3306    // ── minimal mock backend ─────────────────────────────────────────────────
3307
3308    /// Configurable mock backend for tick tests.
3309    struct MockBackend {
3310        /// Tasks returned by `list_by_status(Blocked)`.
3311        blocked_tasks: Vec<ExternalTask>,
3312        /// Sub-issues map: task_id → list of child ExternalIds.
3313        sub_issues: HashMap<String, Vec<ExternalId>>,
3314        /// Tasks returned by `get_task`. Keyed by id.0.
3315        tasks_by_id: HashMap<String, ExternalTask>,
3316        /// Recorded `get_sub_issues` calls.
3317        get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
3318        /// Recorded `update_status` calls.
3319        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
3320        /// Recorded `remove_label` calls.
3321        removed_labels: Arc<Mutex<Vec<(String, String)>>>,
3322        /// Recorded `post_comment` calls.
3323        posted_comments: Arc<Mutex<Vec<(String, String)>>>,
3324    }
3325
3326    impl MockBackend {
3327        fn new() -> Self {
3328            Self {
3329                blocked_tasks: vec![],
3330                sub_issues: HashMap::new(),
3331                tasks_by_id: HashMap::new(),
3332                get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
3333                status_updates: Arc::new(Mutex::new(vec![])),
3334                removed_labels: Arc::new(Mutex::new(vec![])),
3335                posted_comments: Arc::new(Mutex::new(vec![])),
3336            }
3337        }
3338    }
3339
3340    fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
3341        ExternalTask {
3342            id: ExternalId(id.to_string()),
3343            title: format!("Task {id}"),
3344            body: String::new(),
3345            state: "open".to_string(),
3346            labels: labels.iter().map(ToString::to_string).collect(),
3347            author: "bot".to_string(),
3348            created_at: "2026-01-01T00:00:00Z".to_string(),
3349            updated_at: "2026-01-01T00:00:00Z".to_string(),
3350            url: format!("https://github.com/test/test/issues/{id}"),
3351        }
3352    }
3353
3354    #[async_trait]
3355    impl ExternalBackend for MockBackend {
3356        fn name(&self) -> &'static str {
3357            "mock"
3358        }
3359        async fn create_task(
3360            &self,
3361            _title: &str,
3362            _body: &str,
3363            _labels: &[String],
3364        ) -> anyhow::Result<ExternalId> {
3365            Ok(ExternalId("new".to_string()))
3366        }
3367        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3368            self.tasks_by_id
3369                .get(&id.0)
3370                .cloned()
3371                .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
3372        }
3373        async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3374            if status == Status::Blocked {
3375                Ok(self.blocked_tasks.clone())
3376            } else {
3377                Ok(vec![])
3378            }
3379        }
3380        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3381            Ok(vec![])
3382        }
3383        async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
3384            self.posted_comments
3385                .lock()
3386                .unwrap()
3387                .push((_id.0.clone(), _body.to_string()));
3388            Ok(())
3389        }
3390        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
3391            Ok(())
3392        }
3393        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
3394            self.removed_labels
3395                .lock()
3396                .unwrap()
3397                .push((_id.0.clone(), _label.to_string()));
3398            Ok(())
3399        }
3400        async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3401            self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
3402            Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
3403        }
3404        async fn create_sub_task(
3405            &self,
3406            _parent: &ExternalId,
3407            _title: &str,
3408            _body: &str,
3409            _labels: &[String],
3410        ) -> anyhow::Result<ExternalId> {
3411            Ok(ExternalId("child".to_string()))
3412        }
3413        async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
3414            Ok(())
3415        }
3416        async fn has_open_issue_with_title(
3417            &self,
3418            _title: &str,
3419            _label: &str,
3420        ) -> anyhow::Result<bool> {
3421            Ok(false)
3422        }
3423        async fn health_check(&self) -> anyhow::Result<()> {
3424            Ok(())
3425        }
3426        async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
3427            Ok(false)
3428        }
3429        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3430            Ok(Some("testbot".to_string()))
3431        }
3432        async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
3433            Ok(vec![])
3434        }
3435        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3436            self.status_updates
3437                .lock()
3438                .unwrap()
3439                .push((id.0.clone(), status));
3440            Ok(())
3441        }
3442    }
3443
3444    fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
3445        Arc::new(TaskManager::new(backend))
3446    }
3447
3448    // ── tick_unblock_parents ─────────────────────────────────────────────────
3449
3450    #[tokio::test]
3451    async fn unblock_parents_unblocks_when_all_children_done() {
3452        let mut mock = MockBackend::new();
3453
3454        // Blocked parent with two done children
3455        let parent = make_task("10", &["status:blocked"]);
3456        mock.blocked_tasks.push(parent.clone());
3457        mock.sub_issues.insert(
3458            "10".to_string(),
3459            vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
3460        );
3461        mock.tasks_by_id
3462            .insert("11".to_string(), make_task("11", &["status:done"]));
3463        mock.tasks_by_id
3464            .insert("12".to_string(), make_task("12", &["status:done"]));
3465
3466        let status_updates = Arc::clone(&mock.status_updates);
3467        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3468        let task_manager = make_task_manager(Arc::clone(&backend));
3469        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3470
3471        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3472            .await
3473            .unwrap();
3474
3475        let updates = status_updates.lock().unwrap();
3476        assert_eq!(updates.len(), 1, "parent should be unblocked");
3477        assert_eq!(updates[0], ("10".to_string(), Status::New));
3478    }
3479
3480    #[tokio::test]
3481    async fn unblock_parents_skips_when_child_not_done() {
3482        let mut mock = MockBackend::new();
3483
3484        let parent = make_task("20", &["status:blocked"]);
3485        mock.blocked_tasks.push(parent.clone());
3486        mock.sub_issues.insert(
3487            "20".to_string(),
3488            vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3489        );
3490        // Child 21 is done, child 22 is still in_progress
3491        mock.tasks_by_id
3492            .insert("21".to_string(), make_task("21", &["status:done"]));
3493        mock.tasks_by_id
3494            .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3495
3496        let status_updates = Arc::clone(&mock.status_updates);
3497        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3498        let task_manager = make_task_manager(Arc::clone(&backend));
3499        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3500
3501        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3502            .await
3503            .unwrap();
3504
3505        let updates = status_updates.lock().unwrap();
3506        assert!(
3507            updates.is_empty(),
3508            "should not unblock when a child is still running"
3509        );
3510    }
3511
3512    #[tokio::test]
3513    async fn unblock_parents_skips_task_with_no_children() {
3514        let mut mock = MockBackend::new();
3515
3516        // Blocked task with no sub-issues (blocked for a different reason)
3517        let parent = make_task("30", &["status:blocked"]);
3518        mock.blocked_tasks.push(parent.clone());
3519        // No entry in sub_issues → get_sub_issues returns []
3520
3521        let status_updates = Arc::clone(&mock.status_updates);
3522        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3523        let task_manager = make_task_manager(Arc::clone(&backend));
3524        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3525
3526        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3527            .await
3528            .unwrap();
3529
3530        let updates = status_updates.lock().unwrap();
3531        assert!(
3532            updates.is_empty(),
3533            "should not unblock tasks with no sub-issues"
3534        );
3535    }
3536
3537    #[tokio::test]
3538    async fn unblock_parents_skips_task_with_block_reason_in_store() {
3539        // A blocked task that has a known block_reason in the store should never
3540        // trigger a get_sub_issues GraphQL call — it is not waiting on children.
3541        let mut mock = MockBackend::new();
3542
3543        // Blocked parent in the backend list
3544        let parent = make_task("60", &["status:blocked"]);
3545        mock.blocked_tasks.push(parent.clone());
3546        // Give it children in the backend so that, if the skip logic is wrong,
3547        // the parent would get unblocked.
3548        mock.sub_issues
3549            .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3550        mock.tasks_by_id
3551            .insert("61".to_string(), make_task("61", &["status:done"]));
3552
3553        let status_updates = Arc::clone(&mock.status_updates);
3554        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3555        let task_manager = make_task_manager(Arc::clone(&backend));
3556
3557        // Insert the blocked task into the store with a block_reason set.
3558        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3559        let task_id = store
3560            .upsert_external(&crate::store::UpsertExternal {
3561                repo: "test/repo",
3562                ext_id: "60",
3563                title: "Task 60",
3564                body: "",
3565                author: "bot",
3566                url: "https://github.com/test/test/issues/60",
3567                labels: &[],
3568                origin: "github",
3569            })
3570            .await
3571            .unwrap();
3572        store
3573            .update_status(task_id, TaskStatus::Blocked)
3574            .await
3575            .unwrap();
3576        store
3577            .set_block_reason(task_id, Some("CI failure limit reached"))
3578            .await
3579            .unwrap();
3580
3581        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3582            .await
3583            .unwrap();
3584
3585        let updates = status_updates.lock().unwrap();
3586        assert!(
3587            updates.is_empty(),
3588            "task with block_reason should be skipped — no API call, no unblock"
3589        );
3590    }
3591
3592    #[tokio::test]
3593    async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3594        let mock = MockBackend::new();
3595        let get_sub_issues_calls = Arc::clone(&mock.get_sub_issues_calls);
3596        let status_updates = Arc::clone(&mock.status_updates);
3597        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3598        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3599        let task_manager = Arc::new(TaskManager::with_store(
3600            Arc::clone(&backend),
3601            Arc::clone(&store),
3602            "test/repo".to_string(),
3603        ));
3604
3605        let parent_id = store
3606            .create_internal("test/repo", "Parent", "", "job", "daily", None)
3607            .await
3608            .unwrap();
3609        store
3610            .update_status(parent_id, TaskStatus::Blocked)
3611            .await
3612            .unwrap();
3613        let child_id = store
3614            .create(&NewTask {
3615                external_id: Some("91".to_string()),
3616                repo: "test/repo".to_string(),
3617                origin: "github".to_string(),
3618                title: "Child".to_string(),
3619                body: String::new(),
3620                source: String::new(),
3621                source_id: String::new(),
3622                author: String::new(),
3623                url: String::new(),
3624                labels: vec![],
3625                parent_id: Some(parent_id),
3626            })
3627            .await
3628            .unwrap();
3629        store
3630            .update_status(child_id, TaskStatus::Done)
3631            .await
3632            .unwrap();
3633
3634        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3635            .await
3636            .unwrap();
3637
3638        let parent = store.get(parent_id).await.unwrap();
3639        assert_eq!(parent.status, TaskStatus::New);
3640        assert!(
3641            status_updates.lock().unwrap().is_empty(),
3642            "internal parent should be updated in store without backend mirroring"
3643        );
3644        assert!(
3645            get_sub_issues_calls.lock().unwrap().is_empty(),
3646            "internal parent should not query backend sub-issues"
3647        );
3648    }
3649
3650    // ── tick_recover_stuck_tasks (InReview) ──────────────────────────────────
3651
3652    /// Set `updated_at` to a far-past date for a task in an in-memory store.
3653    #[cfg(test)]
3654    async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3655        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3656            .bind(task_id)
3657            .execute(store.pool())
3658            .await
3659            .unwrap();
3660    }
3661
3662    /// Age the in_review status-entry the same way the stale sweep tests do
3663    async fn set_in_review_entered_at_past(store: &TaskStore, task_id: i64) {
3664        sqlx::query(
3665            "UPDATE task_activity SET timestamp = '2020-01-01T00:00:00Z'
3666             WHERE task_id = ? AND to_status = 'in_review'",
3667        )
3668        .bind(task_id)
3669        .execute(store.pool())
3670        .await
3671        .unwrap();
3672    }
3673
3674    #[tokio::test]
3675    async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3676        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3677        let mock = MockBackend::new();
3678        let status_updates = Arc::clone(&mock.status_updates);
3679        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3680        let task_manager = Arc::new(TaskManager::with_store(
3681            Arc::clone(&backend),
3682            Arc::clone(&store),
3683            "owner/repo".to_string(),
3684        ));
3685        let tmux = Arc::new(TmuxManager::new());
3686        let config = EngineConfig {
3687            no_session_stuck_timeout: 600,
3688            stuck_timeout: 1800,
3689            ..EngineConfig::default()
3690        };
3691
3692        // Insert an external task and age both the status entry and updated_at
3693        let id = store
3694            .upsert_external(&crate::store::UpsertExternal {
3695                repo: "owner/repo",
3696                ext_id: "99",
3697                title: "Stuck InReview",
3698                body: "",
3699                author: "",
3700                url: "",
3701                labels: &[],
3702                origin: "github",
3703            })
3704            .await
3705            .unwrap();
3706        store.update_status(id, TaskStatus::InReview).await.unwrap();
3707        set_in_review_entered_at_past(&store, id).await;
3708        set_task_updated_at_past(&store, id).await;
3709
3710        tick_recover_stuck_tasks(
3711            &backend,
3712            &tmux,
3713            "owner/repo",
3714            &task_manager,
3715            &config,
3716            &store,
3717            &HashMap::new(),
3718            &Arc::new(DashMap::new()),
3719        )
3720        .await
3721        .unwrap();
3722
3723        let updates = status_updates.lock().unwrap();
3724        assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3725        assert_eq!(updates[0].0, "99");
3726        assert_eq!(updates[0].1, Status::NeedsReview);
3727    }
3728
3729    #[tokio::test]
3730    async fn recover_stuck_tasks_skips_recent_in_review() {
3731        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3732        let mock = MockBackend::new();
3733        let status_updates = Arc::clone(&mock.status_updates);
3734        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3735        let task_manager = Arc::new(TaskManager::with_store(
3736            Arc::clone(&backend),
3737            Arc::clone(&store),
3738            "owner/repo".to_string(),
3739        ));
3740        let tmux = Arc::new(TmuxManager::new());
3741        let config = EngineConfig {
3742            no_session_stuck_timeout: 600, // 10 minutes — recent task won't trigger
3743            stuck_timeout: 1800,
3744            ..EngineConfig::default()
3745        };
3746
3747        // Age updated_at, leave the in_review status entry at now
3748        let id = store
3749            .upsert_external(&crate::store::UpsertExternal {
3750                repo: "owner/repo",
3751                ext_id: "100",
3752                title: "Recent InReview",
3753                body: "",
3754                author: "",
3755                url: "",
3756                labels: &[],
3757                origin: "github",
3758            })
3759            .await
3760            .unwrap();
3761        store.update_status(id, TaskStatus::InReview).await.unwrap();
3762        set_task_updated_at_past(&store, id).await;
3763
3764        tick_recover_stuck_tasks(
3765            &backend,
3766            &tmux,
3767            "owner/repo",
3768            &task_manager,
3769            &config,
3770            &store,
3771            &HashMap::new(),
3772            &Arc::new(DashMap::new()),
3773        )
3774        .await
3775        .unwrap();
3776
3777        let updates = status_updates.lock().unwrap();
3778        assert!(
3779            updates.is_empty(),
3780            "recent InReview task should not be recovered"
3781        );
3782    }
3783
3784    #[tokio::test]
3785    async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3786        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3787        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3788        let task_manager = Arc::new(TaskManager::with_store(
3789            Arc::clone(&backend),
3790            Arc::clone(&store),
3791            "owner/repo".to_string(),
3792        ));
3793        let tmux = Arc::new(TmuxManager::new());
3794        let config = EngineConfig {
3795            no_session_stuck_timeout: 600,
3796            stuck_timeout: 1800,
3797            ..EngineConfig::default()
3798        };
3799
3800        // Create an internal task in InReview and age both clocks
3801        let id = store
3802            .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3803            .await
3804            .unwrap();
3805        store.update_status(id, TaskStatus::InReview).await.unwrap();
3806        set_in_review_entered_at_past(&store, id).await;
3807        set_task_updated_at_past(&store, id).await;
3808
3809        tick_recover_stuck_tasks(
3810            &backend,
3811            &tmux,
3812            "owner/repo",
3813            &task_manager,
3814            &config,
3815            &store,
3816            &HashMap::new(),
3817            &Arc::new(DashMap::new()),
3818        )
3819        .await
3820        .unwrap();
3821
3822        let task = store.get(id).await.unwrap();
3823        assert_eq!(
3824            task.status,
3825            TaskStatus::NeedsReview,
3826            "stuck internal InReview task should be reset to NeedsReview"
3827        );
3828    }
3829
3830    #[tokio::test]
3831    async fn recover_stuck_tasks_ages_in_review_from_status_entry_not_updated_at() {
3832        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3833        let mock = MockBackend::new();
3834        let status_updates = Arc::clone(&mock.status_updates);
3835        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3836        let task_manager = Arc::new(TaskManager::with_store(
3837            Arc::clone(&backend),
3838            Arc::clone(&store),
3839            "owner/repo".to_string(),
3840        ));
3841        let tmux = Arc::new(TmuxManager::new());
3842        let config = EngineConfig {
3843            no_session_stuck_timeout: 600,
3844            stuck_timeout: 1800,
3845            ..EngineConfig::default()
3846        };
3847
3848        let id = store
3849            .upsert_external(&crate::store::UpsertExternal {
3850                repo: "owner/repo",
3851                ext_id: "102",
3852                title: "Fresh updated_at InReview",
3853                body: "",
3854                author: "",
3855                url: "",
3856                labels: &[],
3857                origin: "github",
3858            })
3859            .await
3860            .unwrap();
3861        store.update_status(id, TaskStatus::InReview).await.unwrap();
3862        set_in_review_entered_at_past(&store, id).await;
3863        store
3864            .set_fields(id, &[("pr_number", serde_json::json!(42))])
3865            .await
3866            .unwrap();
3867
3868        tick_recover_stuck_tasks(
3869            &backend,
3870            &tmux,
3871            "owner/repo",
3872            &task_manager,
3873            &config,
3874            &store,
3875            &HashMap::new(),
3876            &Arc::new(DashMap::new()),
3877        )
3878        .await
3879        .unwrap();
3880
3881        let updates = status_updates.lock().unwrap();
3882        assert_eq!(
3883            updates.len(),
3884            1,
3885            "in_review stuck recovery must age from the status entry even when updated_at is fresh"
3886        );
3887        assert_eq!(updates[0].0, "102");
3888        assert_eq!(updates[0].1, Status::NeedsReview);
3889    }
3890
3891    /// An in_review task whose review session is gone goes back to needs_review, unless its
3892    /// review already recorded a decision and the task is only waiting on the merge
3893    async fn orphaned_review_status_after_recovery(with_decision: bool) -> TaskStatus {
3894        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3895        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3896        let task_manager = Arc::new(TaskManager::with_store(
3897            Arc::clone(&backend),
3898            Arc::clone(&store),
3899            "owner/repo".to_string(),
3900        ));
3901        let tmux = Arc::new(TmuxManager::new());
3902
3903        let id = store
3904            .upsert_external(&crate::store::UpsertExternal {
3905                repo: "owner/repo",
3906                ext_id: "55",
3907                title: "Orphaned review",
3908                body: "",
3909                author: "",
3910                url: "",
3911                labels: &[],
3912                origin: "github",
3913            })
3914            .await
3915            .unwrap();
3916        store.update_status(id, TaskStatus::InReview).await.unwrap();
3917        // Entered in_review long ago. updated_at stays fresh, as review_poll rewrites it
3918        set_in_review_entered_at_past(&store, id).await;
3919        if with_decision {
3920            store::store_log_activity(
3921                &Some(Arc::clone(&store)),
3922                "owner/repo",
3923                "55",
3924                "review_decision",
3925                None,
3926                None,
3927                None::<&str>,
3928                None::<&str>,
3929                Some(&serde_json::json!({"decision": "approve"})),
3930            )
3931            .await;
3932        }
3933
3934        tick_recover_stuck_tasks(
3935            &backend,
3936            &tmux,
3937            "owner/repo",
3938            &task_manager,
3939            &EngineConfig::default(),
3940            &store,
3941            &HashMap::new(),
3942            &Arc::new(DashMap::new()),
3943        )
3944        .await
3945        .unwrap();
3946
3947        store.get(id).await.unwrap().status
3948    }
3949
3950    #[tokio::test]
3951    async fn stale_in_review_recovery_resets_orphaned_task_without_session() {
3952        assert_eq!(
3953            orphaned_review_status_after_recovery(false).await,
3954            TaskStatus::NeedsReview
3955        );
3956    }
3957
3958    #[tokio::test]
3959    async fn stale_in_review_recovery_leaves_decided_task_waiting_on_merge() {
3960        assert_eq!(
3961            orphaned_review_status_after_recovery(true).await,
3962            TaskStatus::InReview
3963        );
3964    }
3965
3966    #[tokio::test]
3967    async fn recover_stuck_tasks_resets_review_agent_failures() {
3968        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3969        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3970        let task_manager = Arc::new(TaskManager::with_store(
3971            Arc::clone(&backend),
3972            Arc::clone(&store),
3973            "owner/repo".to_string(),
3974        ));
3975        let tmux = Arc::new(TmuxManager::new());
3976
3977        let id = store
3978            .upsert_external(&crate::store::UpsertExternal {
3979                repo: "owner/repo",
3980                ext_id: "104",
3981                title: "Orphaned review with failures",
3982                body: "",
3983                author: "",
3984                url: "",
3985                labels: &[],
3986                origin: "github",
3987            })
3988            .await
3989            .unwrap();
3990        store.update_status(id, TaskStatus::InReview).await.unwrap();
3991        store
3992            .set_fields(id, &[("review_agent_failures", serde_json::json!(2))])
3993            .await
3994            .unwrap();
3995        set_in_review_entered_at_past(&store, id).await;
3996
3997        tick_recover_stuck_tasks(
3998            &backend,
3999            &tmux,
4000            "owner/repo",
4001            &task_manager,
4002            &EngineConfig::default(),
4003            &store,
4004            &HashMap::new(),
4005            &Arc::new(DashMap::new()),
4006        )
4007        .await
4008        .unwrap();
4009
4010        let task = store.get(id).await.unwrap();
4011        assert_eq!(task.status, TaskStatus::NeedsReview);
4012        assert_eq!(
4013            task.review_agent_failures, 0,
4014            "a dead review session is an infrastructure event, not an agent failure"
4015        );
4016    }
4017
4018    #[tokio::test]
4019    async fn recover_stuck_tasks_reclaims_hung_in_review_session() {
4020        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4021        let mock = MockBackend::new();
4022        let status_updates = Arc::clone(&mock.status_updates);
4023        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4024        let task_manager = Arc::new(TaskManager::with_store(
4025            Arc::clone(&backend),
4026            Arc::clone(&store),
4027            "owner/repo".to_string(),
4028        ));
4029        let tmux = Arc::new(TmuxManager::new());
4030        let config = EngineConfig {
4031            no_session_stuck_timeout: 600,
4032            stuck_timeout: 1800,
4033            ..EngineConfig::default()
4034        };
4035
4036        let id = store
4037            .upsert_external(&crate::store::UpsertExternal {
4038                repo: "owner/repo",
4039                ext_id: "103",
4040                title: "Hung review session",
4041                body: "",
4042                author: "",
4043                url: "",
4044                labels: &[],
4045                origin: "github",
4046            })
4047            .await
4048            .unwrap();
4049        store.update_status(id, TaskStatus::InReview).await.unwrap();
4050        set_in_review_entered_at_past(&store, id).await;
4051        store
4052            .set_fields(id, &[("pr_number", serde_json::json!(43))])
4053            .await
4054            .unwrap();
4055
4056        let session_name = tmux.session_name("owner/repo", "103-review");
4057        let mut session_map = HashMap::new();
4058        session_map.insert(session_name, true);
4059
4060        tick_recover_stuck_tasks(
4061            &backend,
4062            &tmux,
4063            "owner/repo",
4064            &task_manager,
4065            &config,
4066            &store,
4067            &session_map,
4068            &Arc::new(DashMap::new()),
4069        )
4070        .await
4071        .unwrap();
4072
4073        let updates = status_updates.lock().unwrap();
4074        assert_eq!(
4075            updates.len(),
4076            1,
4077            "a live hung review session must be reclaimed once the in_review entry ages past stuck_timeout"
4078        );
4079        assert_eq!(updates[0].0, "103");
4080        assert_eq!(updates[0].1, Status::NeedsReview);
4081    }
4082
4083    #[tokio::test]
4084    async fn untracked_repo_recovery_resets_stale_in_progress_to_new() {
4085        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4086        let tmux = Arc::new(TmuxManager::new());
4087        let config = EngineConfig {
4088            no_session_stuck_timeout: 600,
4089            stuck_timeout: 1800,
4090            ..EngineConfig::default()
4091        };
4092
4093        // Cross-repo task should still be recovered even when current tick repo is owner/repo.
4094        let id = store
4095            .upsert_external(&crate::store::UpsertExternal {
4096                repo: "owner/other",
4097                ext_id: "777",
4098                title: "Cross-repo stuck task",
4099                body: "",
4100                author: "",
4101                url: "",
4102                labels: &[],
4103                origin: "github",
4104            })
4105            .await
4106            .unwrap();
4107        store
4108            .update_status(id, TaskStatus::InProgress)
4109            .await
4110            .unwrap();
4111        set_task_updated_at_past(&store, id).await;
4112
4113        let active = HashSet::from(["owner/repo".to_string()]);
4114        recover_untracked_repo_stuck_tasks(
4115            &tmux,
4116            &config,
4117            &store,
4118            &HashMap::new(),
4119            &Arc::new(DashMap::new()),
4120            &active,
4121        )
4122        .await;
4123
4124        let task = store.get(id).await.unwrap();
4125        assert_eq!(
4126            task.status,
4127            TaskStatus::New,
4128            "cross-repo stale in_progress task should be reset to New"
4129        );
4130        assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
4131        assert!(task.agent.is_none(), "agent should be cleared");
4132        assert!(task.model.is_none(), "model should be cleared");
4133    }
4134
4135    #[tokio::test]
4136    async fn untracked_repo_recovery_leaves_active_repos_to_their_own_pass() {
4137        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4138        let tmux = Arc::new(TmuxManager::new());
4139        let config = EngineConfig {
4140            no_session_stuck_timeout: 600,
4141            stuck_timeout: 1800,
4142            ..EngineConfig::default()
4143        };
4144        let id = store
4145            .upsert_external(&crate::store::UpsertExternal {
4146                repo: "owner/other",
4147                ext_id: "778",
4148                title: "Stuck task in an active repo",
4149                body: "",
4150                author: "",
4151                url: "",
4152                labels: &[],
4153                origin: "github",
4154            })
4155            .await
4156            .unwrap();
4157        store
4158            .update_status(id, TaskStatus::InProgress)
4159            .await
4160            .unwrap();
4161        set_task_updated_at_past(&store, id).await;
4162
4163        // Per-project passes run concurrently, so only the task's own pass may reclaim it
4164        let active = HashSet::from(["owner/repo".to_string(), "owner/other".to_string()]);
4165        recover_untracked_repo_stuck_tasks(
4166            &tmux,
4167            &config,
4168            &store,
4169            &HashMap::new(),
4170            &Arc::new(DashMap::new()),
4171            &active,
4172        )
4173        .await;
4174
4175        assert_eq!(store.get(id).await.unwrap().status, TaskStatus::InProgress);
4176    }
4177
4178    #[tokio::test]
4179    async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
4180        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4181        let mock = MockBackend::new();
4182        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4183        let task_manager = Arc::new(TaskManager::with_store(
4184            Arc::clone(&backend),
4185            Arc::clone(&store),
4186            "owner/repo".to_string(),
4187        ));
4188        let tmux = Arc::new(TmuxManager::new());
4189        let config = EngineConfig {
4190            no_session_stuck_timeout: 600,
4191            stuck_timeout: 1800,
4192            ..EngineConfig::default()
4193        };
4194
4195        let id = store
4196            .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
4197            .await
4198            .unwrap();
4199        store
4200            .set_fields(
4201                id,
4202                &[
4203                    ("agent", serde_json::json!("claude")),
4204                    ("model", serde_json::json!("sonnet")),
4205                    ("route_attempts", serde_json::json!(1)),
4206                ],
4207            )
4208            .await
4209            .unwrap();
4210        store
4211            .update_status(id, TaskStatus::InProgress)
4212            .await
4213            .unwrap();
4214        set_task_updated_at_past(&store, id).await;
4215
4216        // Start an agent run that will be orphaned by recovery
4217        store
4218            .start_run(&crate::store::StartRun {
4219                task_id: id,
4220                attempt: 1,
4221                run_type: "agent",
4222                agent: "claude",
4223                model: "sonnet",
4224                command: "claude --model sonnet",
4225                prompt: "fix it",
4226            })
4227            .await
4228            .unwrap();
4229
4230        tick_recover_stuck_tasks(
4231            &backend,
4232            &tmux,
4233            "owner/repo",
4234            &task_manager,
4235            &config,
4236            &store,
4237            &HashMap::new(),
4238            &Arc::new(DashMap::new()),
4239        )
4240        .await
4241        .unwrap();
4242
4243        let task = store.get(id).await.unwrap();
4244        assert_eq!(task.status, TaskStatus::New);
4245
4246        let runs = store.get_runs(id).await.unwrap();
4247        assert_eq!(runs.len(), 1);
4248        assert_eq!(runs[0].outcome, "aborted");
4249        assert!(runs[0].completed_at.is_some());
4250        assert_eq!(
4251            runs[0].error,
4252            "stuck-task recovery: internal no session found"
4253        );
4254    }
4255
4256    /// Regression test for #3518: a completed agent run must not be discarded by the
4257    /// stuck-task reclaim just because its tmux session has already exited.
4258    ///
4259    /// In production, sessions are created without `remain-on-exit`, so once the agent
4260    /// process exits tmux removes the session immediately — there is no observable
4261    /// "dead pane" state for `tick_check_session_completions` to catch, and the runner's
4262    /// own `store_touch_updated_at` only fires after its own poll notices the exit (up to
4263    /// its 5s poll interval) plus post-processing. For a task whose total runtime already
4264    /// exceeds `no_session_stuck_timeout`, the very next tick that observes the (now
4265    /// absent) session would otherwise reclaim it instantly — discarding a successful run.
4266    ///
4267    /// The fix: `tick_recover_stuck_tasks` skips the no-session reclaim while the task's
4268    /// dispatch guard (the in-memory `dispatching` map keyed by "{repo}/{task_id}",
4269    /// populated for the entire duration of `runner.run_with_context`) is still held —
4270    /// i.e. while this process is still actively running the task, regardless of tmux
4271    /// session state.
4272    #[tokio::test]
4273    async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
4274        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4275        let mock = MockBackend::new();
4276        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4277        let task_manager = Arc::new(TaskManager::with_store(
4278            Arc::clone(&backend),
4279            Arc::clone(&store),
4280            "owner/repo".to_string(),
4281        ));
4282        let tmux = Arc::new(TmuxManager::new());
4283        let config = EngineConfig {
4284            no_session_stuck_timeout: 600,
4285            stuck_timeout: 1800,
4286            ..EngineConfig::default()
4287        };
4288
4289        let id = store
4290            .create_internal(
4291                "owner/repo",
4292                "Long-running daily job",
4293                "",
4294                "cron",
4295                "1",
4296                None,
4297            )
4298            .await
4299            .unwrap();
4300        store
4301            .set_fields(
4302                id,
4303                &[
4304                    ("agent", serde_json::json!("claude")),
4305                    ("model", serde_json::json!("sonnet")),
4306                    ("route_attempts", serde_json::json!(1)),
4307                ],
4308            )
4309            .await
4310            .unwrap();
4311        store
4312            .update_status(id, TaskStatus::InProgress)
4313            .await
4314            .unwrap();
4315        // Simulate a task that has already run well past no_session_stuck_timeout,
4316        // as a legitimately long agent run would.
4317        set_task_updated_at_past(&store, id).await;
4318
4319        // Simulate the runner still actively processing this task: its dispatch guard
4320        // is held even though the tmux session (empty session_map below) is already gone.
4321        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4322        dispatching.insert(
4323            format!("owner/repo/internal:{id}"),
4324            format!("internal:{id}"),
4325        );
4326
4327        tick_recover_stuck_tasks(
4328            &backend,
4329            &tmux,
4330            "owner/repo",
4331            &task_manager,
4332            &config,
4333            &store,
4334            &HashMap::new(),
4335            &dispatching,
4336        )
4337        .await
4338        .unwrap();
4339
4340        let task = store.get(id).await.unwrap();
4341        assert_eq!(
4342            task.status,
4343            TaskStatus::InProgress,
4344            "task should not be reclaimed while its dispatch guard is still held"
4345        );
4346
4347        // No run should have been finalized/aborted either.
4348        let runs = store.get_runs(id).await.unwrap();
4349        assert!(runs.is_empty());
4350
4351        // Once the guard is released (runner finished), the very same conditions should
4352        // reclaim the task as before — the fix must not disable the safety net entirely.
4353        dispatching.clear();
4354        tick_recover_stuck_tasks(
4355            &backend,
4356            &tmux,
4357            "owner/repo",
4358            &task_manager,
4359            &config,
4360            &store,
4361            &HashMap::new(),
4362            &dispatching,
4363        )
4364        .await
4365        .unwrap();
4366        let task = store.get(id).await.unwrap();
4367        assert_eq!(
4368            task.status,
4369            TaskStatus::New,
4370            "task should still be reclaimed once the dispatch guard is released"
4371        );
4372    }
4373
4374    /// Concurrency regression test for issue #3526: the reclaim race recurred twice
4375    /// (internal:156563 in #3523, internal:156673 in #3526) *after* the #3519/#3525
4376    /// fix landed, in production under the real multi-threaded tokio runtime. The
4377    /// existing `recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held` test
4378    /// above only exercises sequential `.await` calls on a single-threaded test
4379    /// runtime (`#[tokio::test]` defaults to `flavor = "current_thread"`), which can
4380    /// never observe genuine cross-thread races in the shared `dispatching` DashMap.
4381    ///
4382    /// This test claims the guard exactly the way `dispatch_tasks_for_repo` does
4383    /// (atomic `DashMap::entry` claim, `DispatchGuard` RAII release), hands it to a
4384    /// spawned task running on a *different* worker thread that holds it until
4385    /// explicitly told to release, and hammers the real `tick_recover_stuck_tasks`
4386    /// from the main test task hundreds of times while the guard is deterministically
4387    /// still held. If there were a visibility gap between `DashMap::entry().insert()`
4388    /// on one thread and `DashMap::contains_key()` on another, this test would catch it.
4389    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
4390    async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
4391        const HAMMER_ITERATIONS: u32 = 300;
4392
4393        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4394        let mock = MockBackend::new();
4395        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4396        let task_manager = Arc::new(TaskManager::with_store(
4397            Arc::clone(&backend),
4398            Arc::clone(&store),
4399            "owner/repo".to_string(),
4400        ));
4401        let tmux = Arc::new(TmuxManager::new());
4402        let config = EngineConfig {
4403            no_session_stuck_timeout: 600,
4404            stuck_timeout: 1800,
4405            ..EngineConfig::default()
4406        };
4407
4408        let id = store
4409            .create_internal(
4410                "owner/repo",
4411                "Long-running daily job",
4412                "",
4413                "cron",
4414                "1",
4415                None,
4416            )
4417            .await
4418            .unwrap();
4419        store
4420            .set_fields(
4421                id,
4422                &[
4423                    ("agent", serde_json::json!("claude")),
4424                    ("model", serde_json::json!("sonnet")),
4425                    ("route_attempts", serde_json::json!(1)),
4426                ],
4427            )
4428            .await
4429            .unwrap();
4430        store
4431            .update_status(id, TaskStatus::InProgress)
4432            .await
4433            .unwrap();
4434        set_task_updated_at_past(&store, id).await;
4435
4436        let dispatch_key = format!("owner/repo/internal:{id}");
4437        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4438
4439        // Claim the guard synchronously, on this task, exactly like
4440        // dispatch_tasks_for_repo does: the atomic entry-claim always happens before
4441        // tokio::spawn hands the guard to the background task. (Claiming inside the
4442        // spawned task itself would race the hammer loop below against tokio's own
4443        // scheduling of that task onto a worker thread — a test artifact, not the
4444        // real production ordering, which guarantees claim-before-spawn.)
4445        let claim_task_id = format!("internal:{id}");
4446        {
4447            match dispatching.entry(dispatch_key.clone()) {
4448                Entry::Vacant(slot) => {
4449                    slot.insert(claim_task_id);
4450                }
4451                Entry::Occupied(_) => panic!("dispatch key already claimed"),
4452            }
4453        }
4454        let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
4455
4456        // Hold the guard on a spawned task — a different worker thread under the
4457        // multi-thread runtime — mirroring the real runner holding it across genuine
4458        // async work (worktree setup, agent session, response handling) inside
4459        // `tokio::spawn(... run_with_context ...)`. It only drops the guard once
4460        // explicitly told to via `release_rx`, so its lifetime is fully controlled
4461        // by this test rather than raced against a timer — the release-signal
4462        // ordering was itself a source of false positives in an earlier version of
4463        // this test (a `sleep`-then-drop holder can legitimately release the guard
4464        // in the gap between the hammer loop's last check and its next iteration,
4465        // which is a correct outcome, not a bug, but made the assertion flaky).
4466        let (release_tx, release_rx) = oneshot::channel::<()>();
4467        let holder = tokio::spawn(async move {
4468            let _ = release_rx.await;
4469            drop(guard);
4470        });
4471
4472        // Hammer the real reclaim path concurrently from the main task while the
4473        // guard is deterministically still held (the holder task is parked on
4474        // `release_rx` and cannot drop the guard until we signal it below). Each
4475        // iteration's `tick_recover_stuck_tasks` call may itself run on any worker
4476        // thread the tokio scheduler picks, independent of the thread that claimed
4477        // the guard — this is what actually exercises cross-thread DashMap
4478        // visibility, not wall-clock racing against a timer.
4479        for i in 0..HAMMER_ITERATIONS {
4480            tick_recover_stuck_tasks(
4481                &backend,
4482                &tmux,
4483                "owner/repo",
4484                &task_manager,
4485                &config,
4486                &store,
4487                &HashMap::new(),
4488                &dispatching,
4489            )
4490            .await
4491            .unwrap();
4492
4493            let task = store.get(id).await.unwrap();
4494            assert_eq!(
4495                task.status,
4496                TaskStatus::InProgress,
4497                "task must never be reclaimed while the dispatch guard is deterministically \
4498                 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
4499                 iteration={i}, contains_key_now={}, map_len_now={})",
4500                dispatching.contains_key(&dispatch_key),
4501                dispatching.len(),
4502            );
4503            task::yield_now().await;
4504        }
4505
4506        // Now release the guard and confirm the holder task actually completed.
4507        let _ = release_tx.send(());
4508        holder.await.unwrap();
4509
4510        // Guard released — the safety net must still reclaim as before.
4511        tick_recover_stuck_tasks(
4512            &backend,
4513            &tmux,
4514            "owner/repo",
4515            &task_manager,
4516            &config,
4517            &store,
4518            &HashMap::new(),
4519            &dispatching,
4520        )
4521        .await
4522        .unwrap();
4523        let task = store.get(id).await.unwrap();
4524        assert_eq!(
4525            task.status,
4526            TaskStatus::New,
4527            "task should be reclaimed once the dispatch guard is released"
4528        );
4529    }
4530
4531    #[tokio::test]
4532    async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
4533        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4534        let mock = MockBackend::new();
4535        let status_updates = Arc::clone(&mock.status_updates);
4536        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4537        let task_manager = Arc::new(TaskManager::with_store(
4538            Arc::clone(&backend),
4539            Arc::clone(&store),
4540            "owner/repo".to_string(),
4541        ));
4542        let tmux = Arc::new(TmuxManager::new());
4543        let config = EngineConfig {
4544            no_session_stuck_timeout: 600,
4545            stuck_timeout: 1800,
4546            ..EngineConfig::default()
4547        };
4548
4549        let id = store
4550            .upsert_external(&crate::store::UpsertExternal {
4551                repo: "owner/repo",
4552                ext_id: "101",
4553                title: "Waiting For Human Review",
4554                body: "",
4555                author: "",
4556                url: "",
4557                labels: &[],
4558                origin: "github",
4559            })
4560            .await
4561            .unwrap();
4562        store.update_status(id, TaskStatus::InReview).await.unwrap();
4563        set_task_updated_at_past(&store, id).await;
4564
4565        tick_recover_stuck_tasks(
4566            &backend,
4567            &tmux,
4568            "owner/repo",
4569            &task_manager,
4570            &config,
4571            &store,
4572            &HashMap::new(),
4573            &Arc::new(DashMap::new()),
4574        )
4575        .await
4576        .unwrap();
4577
4578        let updates = status_updates.lock().unwrap();
4579        assert!(
4580            updates.is_empty(),
4581            "InReview task waiting for human review should not be reset"
4582        );
4583    }
4584
4585    #[tokio::test]
4586    async fn unblock_parents_no_blocked_tasks() {
4587        let mock = MockBackend::new(); // no blocked tasks
4588        let status_updates = Arc::clone(&mock.status_updates);
4589        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4590        let task_manager = make_task_manager(Arc::clone(&backend));
4591        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4592
4593        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4594            .await
4595            .unwrap();
4596
4597        assert!(status_updates.lock().unwrap().is_empty());
4598    }
4599
4600    #[tokio::test]
4601    async fn unblock_parents_handles_multiple_blocked_tasks() {
4602        let mut mock = MockBackend::new();
4603
4604        // Task 40: all children done → should unblock
4605        let t40 = make_task("40", &["status:blocked"]);
4606        mock.blocked_tasks.push(t40);
4607        mock.sub_issues
4608            .insert("40".to_string(), vec![ExternalId("41".to_string())]);
4609        mock.tasks_by_id
4610            .insert("41".to_string(), make_task("41", &["status:done"]));
4611
4612        // Task 50: child not done → should stay blocked
4613        let t50 = make_task("50", &["status:blocked"]);
4614        mock.blocked_tasks.push(t50);
4615        mock.sub_issues
4616            .insert("50".to_string(), vec![ExternalId("51".to_string())]);
4617        mock.tasks_by_id
4618            .insert("51".to_string(), make_task("51", &["status:in_progress"]));
4619
4620        let status_updates = Arc::clone(&mock.status_updates);
4621        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4622        let task_manager = make_task_manager(Arc::clone(&backend));
4623        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4624
4625        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4626            .await
4627            .unwrap();
4628
4629        let updates = status_updates.lock().unwrap();
4630        assert_eq!(updates.len(), 1, "only one parent should be unblocked");
4631        assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
4632        assert_eq!(updates[0].1, Status::New);
4633    }
4634
4635    /// `tick_all` reads leadership and the global sweeps from the first project's store, so
4636    /// an empty project list must return before any of that, and touch nothing.
4637    #[tokio::test]
4638    async fn tick_all_is_a_no_op_without_projects() {
4639        let transport = Arc::new(Transport::new());
4640        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4641        let tmux = Arc::new(TmuxManager::new());
4642        let semaphore = Arc::new(Semaphore::new(1));
4643        let router = Arc::new(RwLock::new(cooled_review_router_for_test()));
4644        let (weight_tx, mut weight_rx) = mpsc::channel(8);
4645        let dispatching = Arc::new(DashMap::new());
4646        let auto_merge_in_flight = Arc::new(DashSet::new());
4647
4648        tick_all(
4649            &[],
4650            &tmux,
4651            &capture,
4652            &semaphore,
4653            &EngineConfig::default(),
4654            &router,
4655            &weight_tx,
4656            &dispatching,
4657            Some(&transport),
4658            &HashSet::new(),
4659            &auto_merge_in_flight,
4660        )
4661        .await;
4662
4663        assert!(dispatching.is_empty(), "no project, so no dispatch claimed");
4664        assert!(
4665            weight_rx.try_recv().is_err(),
4666            "no project, so no weight signal"
4667        );
4668    }
4669
4670    /// The store-wide sweeps must run once per tick, not once per project. The NeedsReview
4671    /// refire sweep reads `needs_review_refires` and then increments it, so when it ran
4672    /// inside the per-project join every project saw the same count and spent one of the
4673    /// five refire attempts — four projects burned four attempts in a single tick (#3864).
4674    #[serial_test::serial(cooldown_state)]
4675    #[tokio::test]
4676    async fn tick_all_runs_the_global_needs_review_sweep_once_for_two_projects() {
4677        cooldown::reset_global_state().await;
4678
4679        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4680        let id = store
4681            .upsert_external(&UpsertExternal {
4682                repo: "owner/one",
4683                ext_id: "7",
4684                title: "Stale needs_review task",
4685                body: "",
4686                author: "user",
4687                url: "https://github.com/owner/one/issues/7",
4688                labels: &[],
4689                origin: "github",
4690            })
4691            .await
4692            .unwrap();
4693        store
4694            .update_status(id, TaskStatus::NeedsReview)
4695            .await
4696            .unwrap();
4697        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4698            .bind(id)
4699            .execute(store.pool())
4700            .await
4701            .unwrap();
4702
4703        let engines: Vec<ProjectEngine> = ["owner/one", "owner/two"]
4704            .iter()
4705            .map(|repo| {
4706                let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4707                ProjectEngine {
4708                    repo: repo.to_string(),
4709                    project_dir: PathBuf::from("/nonexistent"),
4710                    backend: Arc::clone(&backend),
4711                    task_manager: Arc::new(TaskManager::with_store(
4712                        backend,
4713                        Arc::clone(&store),
4714                        repo.to_string(),
4715                    )),
4716                    runner: Arc::new(TaskRunner::new(repo.to_string())),
4717                    store: Arc::clone(&store),
4718                }
4719            })
4720            .collect();
4721
4722        // A healthy review agent, so the sweep does not skip the pass
4723        let mut config = RouterConfig::default();
4724        config
4725            .model_map
4726            .entry("review".to_string())
4727            .or_default()
4728            .insert("claude".to_string(), vec!["sonnet".to_string()]);
4729        let router = Arc::new(RwLock::new(Router::new_for_test(
4730            config,
4731            vec!["claude".to_string()],
4732        )));
4733
4734        let transport = Arc::new(Transport::new());
4735        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4736        let (weight_tx, _weight_rx) = mpsc::channel(8);
4737
4738        tick_all(
4739            &engines,
4740            &Arc::new(TmuxManager::new()),
4741            &capture,
4742            &Arc::new(Semaphore::new(1)),
4743            &EngineConfig::default(),
4744            &router,
4745            &weight_tx,
4746            &Arc::new(DashMap::new()),
4747            Some(&transport),
4748            &engines.iter().map(|e| e.repo.clone()).collect(),
4749            &Arc::new(DashSet::new()),
4750        )
4751        .await;
4752
4753        assert_eq!(
4754            store.get(id).await.unwrap().needs_review_refires,
4755            1,
4756            "two projects must share one pass of the global NeedsReview sweep"
4757        );
4758    }
4759
4760    /// `tick_all` must not reap tmux sessions it did not start. Startup cleanup used to
4761    /// live here, and nextest's per-test process made the once-flag kill live agents (#3889)
4762    #[tokio::test]
4763    async fn tick_all_does_not_kill_sessions_it_did_not_create() {
4764        let tmux = Arc::new(TmuxManager::new());
4765        let Ok(session_name) = tmux
4766            .create_session("other/foreign", "3889", "/tmp", "sleep 30", &[])
4767            .await
4768        else {
4769            eprintln!("Skipping test: tmux not available or failed to create test session");
4770            return;
4771        };
4772        let _guard = SessionGuard(session_name.clone());
4773
4774        // Cross a unix-second boundary so a reintroduced startup_cleanup would treat
4775        // this session as stale (created_at < now)
4776        time::sleep(Duration::from_millis(1100)).await;
4777
4778        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4779        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4780        let engines = vec![ProjectEngine {
4781            repo: "owner/one".to_string(),
4782            project_dir: PathBuf::from("/nonexistent"),
4783            backend: Arc::clone(&backend),
4784            task_manager: Arc::new(TaskManager::with_store(
4785                backend,
4786                Arc::clone(&store),
4787                "owner/one".to_string(),
4788            )),
4789            runner: Arc::new(TaskRunner::new("owner/one".to_string())),
4790            store,
4791        }];
4792
4793        let transport = Arc::new(Transport::new());
4794        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
4795        let (weight_tx, _weight_rx) = mpsc::channel(8);
4796        let router = Arc::new(RwLock::new(cooled_review_router_for_test()));
4797
4798        tick_all(
4799            &engines,
4800            &tmux,
4801            &capture,
4802            &Arc::new(Semaphore::new(1)),
4803            &EngineConfig::default(),
4804            &router,
4805            &weight_tx,
4806            &Arc::new(DashMap::new()),
4807            Some(&transport),
4808            &engines.iter().map(|e| e.repo.clone()).collect(),
4809            &Arc::new(DashSet::new()),
4810        )
4811        .await;
4812
4813        assert!(
4814            tmux.session_exists(&session_name).await,
4815            "tick_all must not kill a tmux session it did not start"
4816        );
4817    }
4818
4819    #[tokio::test]
4820    async fn detect_silent_agents_skips_backend_for_internal_tasks() {
4821        let transport = Arc::new(Transport::new());
4822        let capture = Arc::new(CaptureService::new(transport));
4823        let mock = Arc::new(MockBackend::new());
4824        let backend: Arc<dyn ExternalBackend> = Arc::<MockBackend>::clone(&mock);
4825        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4826        let task_manager = Arc::new(TaskManager::with_store(
4827            Arc::clone(&backend),
4828            Arc::clone(&store),
4829            "owner/repo".to_string(),
4830        ));
4831        let tmux = Arc::new(TmuxManager::new());
4832        let config = EngineConfig {
4833            silence_grace_period: 0,
4834            ..EngineConfig::default()
4835        };
4836
4837        let internal_id = store
4838            .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
4839            .await
4840            .unwrap();
4841        store
4842            .update_status(internal_id, TaskStatus::InProgress)
4843            .await
4844            .unwrap();
4845
4846        let task_id = format!("internal:{internal_id}");
4847        capture
4848            .register_session("owner/repo", &task_id, "orch-test-internal")
4849            .await;
4850        time::sleep(Duration::from_millis(1100)).await;
4851
4852        tick_detect_silent_agents(
4853            &tmux,
4854            "owner/repo",
4855            &capture,
4856            &backend,
4857            &task_manager,
4858            &config,
4859            &store,
4860        )
4861        .await
4862        .unwrap();
4863
4864        assert!(
4865            mock.removed_labels.lock().unwrap().is_empty(),
4866            "internal tasks should skip backend label removal"
4867        );
4868        assert!(
4869            mock.posted_comments.lock().unwrap().is_empty(),
4870            "internal tasks should skip backend comments"
4871        );
4872
4873        let task = store.get(internal_id).await.unwrap();
4874        // Silence detection now uses failover: if a fallback agent is found → Routed,
4875        // otherwise → New (reset for re-routing when cooldowns expire). Either is
4876        // acceptable; the key invariant is that the task is no longer InProgress.
4877        assert!(
4878            matches!(task.status, TaskStatus::Routed | TaskStatus::New),
4879            "internal task should be routed to fallback or reset to new, got {:?}",
4880            task.status
4881        );
4882    }
4883
4884    #[tokio::test]
4885    async fn detect_silent_agents_finalizes_open_task_runs_rows() {
4886        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4887        let mock = MockBackend::new();
4888        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4889        let task_manager = Arc::new(TaskManager::with_store(
4890            Arc::clone(&backend),
4891            Arc::clone(&store),
4892            "owner/repo".to_string(),
4893        ));
4894        let transport = Arc::new(Transport::new());
4895        let capture = Arc::new(CaptureService::new(transport));
4896        let tmux = Arc::new(TmuxManager::new());
4897        let config = EngineConfig {
4898            silence_grace_period: 60,
4899            silence_cooldown: 300,
4900            ..EngineConfig::default()
4901        };
4902
4903        let internal_id = store
4904            .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4905            .await
4906            .unwrap();
4907        store
4908            .set_fields(
4909                internal_id,
4910                &[
4911                    ("agent", serde_json::json!("claude")),
4912                    ("model", serde_json::json!("sonnet")),
4913                ],
4914            )
4915            .await
4916            .unwrap();
4917        store
4918            .update_status(internal_id, TaskStatus::InProgress)
4919            .await
4920            .unwrap();
4921
4922        let task_id = format!("internal:{internal_id}");
4923        capture
4924            .register_session("owner/repo", &task_id, "orch-test-silent-run")
4925            .await;
4926        capture
4927            .set_buffer_state_for_test(
4928                "owner/repo",
4929                &task_id,
4930                false,
4931                false,
4932                chrono::Utc::now() - chrono::Duration::seconds(120),
4933            )
4934            .await;
4935
4936        // Start an agent run that will be orphaned by silence detection
4937        store
4938            .start_run(&crate::store::StartRun {
4939                task_id: internal_id,
4940                attempt: 1,
4941                run_type: "agent",
4942                agent: "claude",
4943                model: "sonnet",
4944                command: "claude --model sonnet",
4945                prompt: "fix it",
4946            })
4947            .await
4948            .unwrap();
4949
4950        tick_detect_silent_agents(
4951            &tmux,
4952            "owner/repo",
4953            &capture,
4954            &backend,
4955            &task_manager,
4956            &config,
4957            &store,
4958        )
4959        .await
4960        .unwrap();
4961
4962        let runs = store.get_runs(internal_id).await.unwrap();
4963        assert_eq!(runs.len(), 1);
4964        assert_eq!(runs[0].outcome, "aborted");
4965        assert!(runs[0].completed_at.is_some());
4966        assert_eq!(runs[0].error, "silence detection: session killed");
4967    }
4968
4969    /// Regression test for the race between Phase 1 (tick_check_session_completions)
4970    /// and Phase 2 (tick_recover_stuck_tasks).
4971    ///
4972    /// Before the fix, Phase 1 killed the tmux session without touching updated_at.
4973    /// Phase 2 could then see status=in_progress, has_session=false, and an old
4974    /// updated_at, and would reset the task to New — even though the runner's
4975    /// wait_for_completion() poll had not yet returned.
4976    ///
4977    /// The fix: Phase 1 touches updated_at as soon as it detects the session is done.
4978    /// This test verifies that after the touch, Phase 2 does NOT reclaim the task.
4979    #[tokio::test]
4980    async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4981        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4982        let mock = MockBackend::new();
4983        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4984        let task_manager = Arc::new(TaskManager::with_store(
4985            Arc::clone(&backend),
4986            Arc::clone(&store),
4987            "owner/repo".to_string(),
4988        ));
4989        let tmux = Arc::new(TmuxManager::new());
4990        let config = EngineConfig {
4991            no_session_stuck_timeout: 600,
4992            stuck_timeout: 1800,
4993            ..EngineConfig::default()
4994        };
4995
4996        // Create an in-progress internal task with old updated_at.
4997        // Without the fix this task would be reclaimed by Phase 2 (age >> threshold,
4998        // no session running in the test environment).
4999        let id = store
5000            .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
5001            .await
5002            .unwrap();
5003        store
5004            .update_status(id, TaskStatus::InProgress)
5005            .await
5006            .unwrap();
5007        set_task_updated_at_past(&store, id).await;
5008
5009        // Simulate Phase 1: touch updated_at when the session is detected as done.
5010        // In production this happens inside tick_check_session_completions.
5011        let task_id = format!("internal:{id}");
5012        store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
5013
5014        // Phase 2 runs — updated_at is fresh so the task must NOT be reclaimed.
5015        tick_recover_stuck_tasks(
5016            &backend,
5017            &tmux,
5018            "owner/repo",
5019            &task_manager,
5020            &config,
5021            &store,
5022            &HashMap::new(),
5023            &Arc::new(DashMap::new()),
5024        )
5025        .await
5026        .unwrap();
5027
5028        let task = store.get(id).await.unwrap();
5029        assert_eq!(
5030            task.status,
5031            TaskStatus::InProgress,
5032            "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
5033        );
5034    }
5035
5036    #[tokio::test]
5037    async fn stuck_task_timing_treats_dead_session_as_missing() {
5038        let tmux = Arc::new(TmuxManager::new());
5039        let repo = "owner/repo";
5040        let task_id = "internal:33498";
5041        let session_name = tmux.session_name(repo, task_id);
5042
5043        let create_result = tmux
5044            .command()
5045            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5046            .output()
5047            .await;
5048
5049        match create_result {
5050            Ok(o) if o.status.success() => {}
5051            _ => {
5052                eprintln!("Skipping test: tmux not available or failed to create test session");
5053                return;
5054            }
5055        }
5056
5057        let _guard = SessionGuard(session_name.clone());
5058
5059        let set_option_result = tmux
5060            .command()
5061            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5062            .output()
5063            .await;
5064        if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
5065            eprintln!("Skipping test: unable to set tmux remain-on-exit option");
5066            return;
5067        }
5068
5069        let send_exit_result = tmux
5070            .command()
5071            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5072            .output()
5073            .await;
5074        if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
5075            eprintln!("Skipping test: unable to exit tmux pane");
5076            return;
5077        }
5078
5079        // Poll until pane is dead (up to 2s) — fixed sleep is unreliable under load.
5080        for _ in 0..20 {
5081            time::sleep(Duration::from_millis(100)).await;
5082            if !tmux.session_is_running(&session_name).await {
5083                break;
5084            }
5085        }
5086
5087        // Also wait for batch_session_active to reflect the dead state.
5088        // session_is_running uses `list-panes -t <session>` which may settle
5089        // before the global `list-panes -a` snapshot used by the production code.
5090        // On CI under load these two tmux commands can transiently disagree.
5091        let mut session_map = tmux.batch_session_active().await;
5092        for _ in 0..20 {
5093            if !session_map.get(&session_name).copied().unwrap_or(false) {
5094                break;
5095            }
5096            time::sleep(Duration::from_millis(100)).await;
5097            session_map = tmux.batch_session_active().await;
5098        }
5099        if session_map.get(&session_name).copied().unwrap_or(false) {
5100            eprintln!(
5101                "Skipping test: session still alive in batch_session_active after retries (CI tmux lag)"
5102            );
5103            return;
5104        }
5105
5106        let config = EngineConfig {
5107            no_session_stuck_timeout: 600,
5108            stuck_timeout: 1800,
5109            ..EngineConfig::default()
5110        };
5111        let updated_at = (chrono::Utc::now()
5112            - chrono::Duration::seconds(
5113                i64::try_from(config.no_session_stuck_timeout).expect("timeout fits in i64") + 5,
5114            ))
5115        .to_rfc3339();
5116
5117        let timing = stuck_task_timing_from_map(
5118            &tmux,
5119            repo,
5120            task_id,
5121            &updated_at,
5122            &config,
5123            "parse failure",
5124            &session_map,
5125        )
5126        .expect("dead session should be treated as missing");
5127
5128        assert!(!timing.has_session);
5129        assert_eq!(timing.threshold, config.no_session_stuck_timeout);
5130    }
5131
5132    #[tokio::test]
5133    #[serial_test::serial(suspend_state)]
5134    async fn stuck_task_timing_discounts_host_suspend_gap() {
5135        super::super::suspend::clear_for_test();
5136
5137        let tmux = Arc::new(TmuxManager::new());
5138        let repo = "owner/repo";
5139        let task_id = "internal:99001";
5140        let session_map = HashMap::new();
5141
5142        let config = EngineConfig {
5143            no_session_stuck_timeout: 1800,
5144            stuck_timeout: 1800,
5145            ..EngineConfig::default()
5146        };
5147
5148        // Task was last touched 33 minutes ago (age_mins=33 matches the reported
5149        // false-positive), but the host was suspended for 30 of those minutes —
5150        // real elapsed runtime is only 3 minutes, well under the 30-minute threshold.
5151        let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
5152        let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
5153        super::super::suspend::inject_gap_for_test(
5154            suspend_gap_detected_at,
5155            chrono::Duration::minutes(30),
5156        );
5157
5158        let timing = stuck_task_timing_from_map(
5159            &tmux,
5160            repo,
5161            task_id,
5162            &updated_at,
5163            &config,
5164            "parse failure",
5165            &session_map,
5166        );
5167
5168        assert!(
5169            timing.is_none(),
5170            "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
5171        );
5172
5173        super::super::suspend::clear_for_test();
5174    }
5175
5176    #[tokio::test]
5177    async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
5178        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5179        let mock = MockBackend::new();
5180        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5181        let task_manager = Arc::new(TaskManager::with_store(
5182            Arc::clone(&backend),
5183            Arc::clone(&store),
5184            "owner/repo".to_string(),
5185        ));
5186        let tmux = Arc::new(TmuxManager::new());
5187        let config = EngineConfig {
5188            no_session_stuck_timeout: 600,
5189            stuck_timeout: 1,
5190            ..EngineConfig::default()
5191        };
5192
5193        let id = store
5194            .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
5195            .await
5196            .unwrap();
5197        store.update_status(id, TaskStatus::Routed).await.unwrap();
5198        set_task_updated_at_past(&store, id).await;
5199
5200        let task_id = format!("internal:{id}");
5201        let session_name = tmux.session_name("owner/repo", &task_id);
5202        let create_result = tmux
5203            .command()
5204            .args([
5205                "new-session",
5206                "-d",
5207                "-s",
5208                &session_name,
5209                "-c",
5210                "/tmp",
5211                "sleep 60",
5212            ])
5213            .output()
5214            .await;
5215
5216        match create_result {
5217            Ok(o) if o.status.success() => {}
5218            _ => {
5219                eprintln!("Skipping test: tmux not available or failed to create test session");
5220                return;
5221            }
5222        }
5223
5224        let session_map = tmux.batch_session_active().await;
5225        assert_eq!(session_map.get(&session_name).copied(), Some(true));
5226
5227        tick_recover_stuck_tasks(
5228            &backend,
5229            &tmux,
5230            "owner/repo",
5231            &task_manager,
5232            &config,
5233            &store,
5234            &session_map,
5235            &Arc::new(DashMap::new()),
5236        )
5237        .await
5238        .unwrap();
5239
5240        let task = store.get(id).await.unwrap();
5241        assert_eq!(
5242            task.status,
5243            TaskStatus::New,
5244            "routed task blocked by a stale session should be reset for re-routing"
5245        );
5246        assert!(
5247            !tmux.session_exists(&session_name).await,
5248            "recovery should kill the stale tmux session so dispatch can proceed"
5249        );
5250    }
5251
5252    /// An internal complex task that entered in_progress `minutes` ago and was last written
5253    /// `updated_minutes_ago`, with a live tmux session. None when tmux is not available
5254    async fn live_complex_run(
5255        store: &TaskStore,
5256        tmux: &TmuxManager,
5257        minutes: i64,
5258        updated_minutes_ago: i64,
5259    ) -> Option<(i64, String)> {
5260        let id = store
5261            .create_internal("owner/repo", "Long complex run", "", "cron", "1", None)
5262            .await
5263            .unwrap();
5264        store
5265            .set_fields(id, &[("complexity", serde_json::json!("complex"))])
5266            .await
5267            .unwrap();
5268        store
5269            .update_status(id, TaskStatus::InProgress)
5270            .await
5271            .unwrap();
5272        let ago = |m: i64| {
5273            (chrono::Utc::now() - chrono::Duration::minutes(m))
5274                .format("%Y-%m-%dT%H:%M:%SZ")
5275                .to_string()
5276        };
5277        let entered = ago(minutes);
5278        sqlx::query(
5279            "UPDATE task_activity SET timestamp = ? WHERE task_id = ? AND to_status = 'in_progress'",
5280        )
5281        .bind(&entered)
5282        .bind(id)
5283        .execute(store.pool())
5284        .await
5285        .unwrap();
5286        sqlx::query("UPDATE tasks SET updated_at = ? WHERE id = ?")
5287            .bind(ago(updated_minutes_ago))
5288            .bind(id)
5289            .execute(store.pool())
5290            .await
5291            .unwrap();
5292
5293        let session_name = tmux.session_name("owner/repo", &format!("internal:{id}"));
5294        let created = tmux
5295            .command()
5296            .args([
5297                "new-session",
5298                "-d",
5299                "-s",
5300                &session_name,
5301                "-c",
5302                "/tmp",
5303                "sleep 60",
5304            ])
5305            .output()
5306            .await;
5307        match created {
5308            Ok(o) if o.status.success() => Some((id, session_name)),
5309            _ => None,
5310        }
5311    }
5312
5313    fn long_run_config() -> EngineConfig {
5314        EngineConfig {
5315            stuck_timeout: 1800,
5316            run_timeouts: RunTimeouts {
5317                base: 1800,
5318                medium: 2700,
5319                complex: 3600,
5320            },
5321            ..EngineConfig::default()
5322        }
5323    }
5324
5325    #[test]
5326    fn tick_router_loads_the_config_once_for_many_live_runs() {
5327        let cell = OnceLock::new();
5328        let loads = Cell::new(0);
5329        for _ in 0..4 {
5330            tick_router(&cell, || {
5331                loads.set(loads.get() + 1);
5332                RouterConfig::default()
5333            });
5334        }
5335        assert_eq!(loads.get(), 1);
5336    }
5337
5338    #[tokio::test]
5339    async fn run_tier_follows_the_tier_of_the_stored_model() {
5340        let mut router = RouterConfig::default();
5341        router.model_map.clear();
5342        for (tier, model) in [("simple", "rt-s"), ("complex", "rt-c")] {
5343            router
5344                .model_map
5345                .entry(tier.to_string())
5346                .or_default()
5347                .insert("rtagent".to_string(), vec![model.to_string()]);
5348        }
5349        let store = TaskStore::open_memory().await.unwrap();
5350        let id = store
5351            .create_internal(
5352                "owner/repo",
5353                "Simple on a complex model",
5354                "",
5355                "cron",
5356                "1",
5357                None,
5358            )
5359            .await
5360            .unwrap();
5361        store
5362            .set_fields(
5363                id,
5364                &[
5365                    ("complexity", serde_json::json!("simple")),
5366                    ("agent", serde_json::json!("rtagent")),
5367                    ("model", serde_json::json!("rt-c")),
5368                ],
5369            )
5370            .await
5371            .unwrap();
5372        let task = store.get(id).await.unwrap();
5373        assert_eq!(run_tier(&router, &task), "complex");
5374
5375        store
5376            .set_fields(id, &[("model", serde_json::json!("rt-s"))])
5377            .await
5378            .unwrap();
5379        let task = store.get(id).await.unwrap();
5380        assert_eq!(run_tier(&router, &task), "simple");
5381    }
5382
5383    #[tokio::test]
5384    async fn recover_stuck_tasks_keeps_a_live_complex_run_inside_its_timeout() {
5385        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5386        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5387        let task_manager = Arc::new(TaskManager::with_store(
5388            Arc::clone(&backend),
5389            Arc::clone(&store),
5390            "owner/repo".to_string(),
5391        ));
5392        let tmux = Arc::new(TmuxManager::new());
5393        let Some((id, session_name)) = live_complex_run(&store, &tmux, 31, 31).await else {
5394            eprintln!("Skipping test: tmux not available or failed to create test session");
5395            return;
5396        };
5397        let session_map = tmux.batch_session_active().await;
5398
5399        tick_recover_stuck_tasks(
5400            &backend,
5401            &tmux,
5402            "owner/repo",
5403            &task_manager,
5404            &long_run_config(),
5405            &store,
5406            &session_map,
5407            &Arc::new(DashMap::new()),
5408        )
5409        .await
5410        .unwrap();
5411
5412        let alive = tmux.session_exists(&session_name).await;
5413        let _ = tmux.kill_session(&session_name).await;
5414        assert_eq!(
5415            store.get(id).await.unwrap().status,
5416            TaskStatus::InProgress,
5417            "a complex run at 31 minutes is inside its 3600s timeout"
5418        );
5419        assert!(alive, "recovery must not touch a run inside its timeout");
5420    }
5421
5422    #[tokio::test]
5423    async fn recover_stuck_tasks_kills_a_live_complex_run_past_its_timeout() {
5424        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5425        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5426        let task_manager = Arc::new(TaskManager::with_store(
5427            Arc::clone(&backend),
5428            Arc::clone(&store),
5429            "owner/repo".to_string(),
5430        ));
5431        let tmux = Arc::new(TmuxManager::new());
5432        let Some((id, session_name)) = live_complex_run(&store, &tmux, 66, 0).await else {
5433            eprintln!("Skipping test: tmux not available or failed to create test session");
5434            return;
5435        };
5436        let session_map = tmux.batch_session_active().await;
5437
5438        tick_recover_stuck_tasks(
5439            &backend,
5440            &tmux,
5441            "owner/repo",
5442            &task_manager,
5443            &long_run_config(),
5444            &store,
5445            &session_map,
5446            &Arc::new(DashMap::new()),
5447        )
5448        .await
5449        .unwrap();
5450
5451        let alive = tmux.session_exists(&session_name).await;
5452        let _ = tmux.kill_session(&session_name).await;
5453        assert_eq!(
5454            store.get(id).await.unwrap().status,
5455            TaskStatus::New,
5456            "a complex run past its timeout plus grace is stuck"
5457        );
5458        assert!(
5459            !alive,
5460            "recovery must kill the session it reports as killed"
5461        );
5462    }
5463
5464    // ── tick_dispatch_tasks: deadlock regression test (#1361) ──────────────
5465
5466    /// Regression test for #1361: tick_dispatch_tasks previously accepted
5467    /// `&Arc<RwLock<Router>>` and called `router_arc.read().await` internally.
5468    /// The main loop already held a write lock on the same RwLock, so the read
5469    /// could never be acquired — guaranteed deadlock on every tick with routed
5470    /// tasks.
5471    ///
5472    /// The fix changed the signature to accept `&Router` directly (the caller
5473    /// already has the dereferenced guard), eliminating the lock re-acquisition.
5474    ///
5475    /// This test verifies the function completes within 2 seconds when the
5476    /// caller holds the write lock — the old code would deadlock here.
5477    #[tokio::test]
5478    async fn dispatch_does_not_deadlock_under_write_lock() {
5479        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5480        let mock = MockBackend::new();
5481        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5482        let task_manager = Arc::new(TaskManager::with_store(
5483            Arc::clone(&backend),
5484            Arc::clone(&store),
5485            "owner/repo".to_string(),
5486        ));
5487        let tmux = Arc::new(TmuxManager::new());
5488        let semaphore = Arc::new(Semaphore::new(4));
5489        let (weight_tx, _weight_rx) = mpsc::channel(16);
5490        let dispatching = Arc::new(DashMap::new());
5491        let transport = Arc::new(Transport::new());
5492        let capture = Arc::new(CaptureService::new(transport));
5493
5494        let router_config = RouterConfig::default();
5495        let router = Router::new_for_test(router_config, vec!["claude".to_string()]);
5496        let router_arc = Arc::new(RwLock::new(router));
5497
5498        // Simulate what the main loop does: hold a write lock, snapshot
5499        // dispatch mode, then call tick_dispatch_tasks without passing any
5500        // lock guard into async dispatch work.
5501        let write_guard = router_arc.write().await;
5502        let dispatch_mode = dispatch_mode_from_router(&write_guard);
5503
5504        let result = time::timeout(
5505            Duration::from_secs(2),
5506            tick_dispatch_tasks(
5507                &backend,
5508                &tmux,
5509                "owner/repo",
5510                &Arc::new(TaskRunner::new("owner/repo".to_string())),
5511                &capture,
5512                &semaphore,
5513                &task_manager,
5514                &weight_tx,
5515                dispatch_mode,
5516                &dispatching,
5517                &store,
5518                &HashMap::new(),
5519            ),
5520        )
5521        .await;
5522
5523        assert!(
5524            result.is_ok(),
5525            "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
5526             on router_arc while a write lock was already held (issue #1361)"
5527        );
5528    }
5529
5530    /// Regression test for #3599: a task routed to an agent/model that entered
5531    /// cooldown *after* routing but *before* dispatch must not be dispatched —
5532    /// the stale route was decided before the failure that triggered the
5533    /// cooldown was known. Dispatch must re-check the assigned agent/model
5534    /// against the same cooldown store used at routing time and, if stale,
5535    /// leave the task in `Routed` for the next tick to re-evaluate (no PR is
5536    /// spawned, no per-task defer timer is written).
5537    #[tokio::test]
5538    async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
5539        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5540        let mock = MockBackend::new();
5541        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5542        let task_manager = Arc::new(TaskManager::with_store(
5543            Arc::clone(&backend),
5544            Arc::clone(&store),
5545            "owner/repo".to_string(),
5546        ));
5547        let tmux = Arc::new(TmuxManager::new());
5548        let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5549        let semaphore = Arc::new(Semaphore::new(4));
5550        let (weight_tx, _weight_rx) = mpsc::channel(16);
5551        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5552        let transport = Arc::new(Transport::new());
5553        let capture = Arc::new(CaptureService::new(transport));
5554
5555        let id = store
5556            .upsert_external(&UpsertExternal {
5557                repo: "owner/repo",
5558                ext_id: "999",
5559                title: "Stale routed task",
5560                body: "",
5561                author: "user",
5562                url: "https://github.com/owner/repo/issues/999",
5563                labels: &[],
5564                origin: "github",
5565            })
5566            .await
5567            .unwrap();
5568        store
5569            .store_route(&StoreRoute {
5570                id,
5571                agent: "minimax",
5572                model: Some("opus"),
5573                complexity: "medium",
5574                estimate: 0,
5575                reason: "test",
5576                profile: "{}",
5577            })
5578            .await
5579            .unwrap();
5580        store.update_status(id, DbStatus::Routed).await.unwrap();
5581
5582        // Simulate a billing/rate-limit cooldown recorded for minimax:opus after
5583        // this task was routed but before it dispatched.
5584        cooldown::set_model_cooldown("minimax", "opus", 300, "rate_limit_usage_window").await;
5585
5586        let dispatch_mode = DispatchMode {
5587            is_degraded: false,
5588            healthy_agents: 1,
5589            threshold: 1,
5590        };
5591
5592        tick_dispatch_tasks(
5593            &backend,
5594            &tmux,
5595            "owner/repo",
5596            &runner,
5597            &capture,
5598            &semaphore,
5599            &task_manager,
5600            &weight_tx,
5601            dispatch_mode,
5602            &dispatching,
5603            &store,
5604            &HashMap::new(),
5605        )
5606        .await
5607        .unwrap();
5608
5609        let task = store.get(id).await.unwrap();
5610        assert_eq!(
5611            task.status,
5612            DbStatus::Routed,
5613            "task whose routed model is now cooled down must stay Routed, not be dispatched"
5614        );
5615        assert!(
5616            dispatching.is_empty(),
5617            "dispatch guard must be released when skipping a stale-cooldown task"
5618        );
5619    }
5620
5621    #[tokio::test]
5622    async fn dispatch_skips_a_task_with_an_open_dependency() {
5623        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5624        let mock = MockBackend::new();
5625        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5626        let task_manager = Arc::new(TaskManager::with_store(
5627            Arc::clone(&backend),
5628            Arc::clone(&store),
5629            "owner/repo".to_string(),
5630        ));
5631        let tmux = Arc::new(TmuxManager::new());
5632        let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5633        let semaphore = Arc::new(Semaphore::new(4));
5634        let (weight_tx, _weight_rx) = mpsc::channel(16);
5635        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5636        let transport = Arc::new(Transport::new());
5637        let capture = Arc::new(CaptureService::new(transport));
5638
5639        let id = store
5640            .upsert_external(&UpsertExternal {
5641                repo: "owner/repo",
5642                ext_id: "42",
5643                title: "Waiting routed task",
5644                body: "",
5645                author: "user",
5646                url: "https://github.com/owner/repo/issues/42",
5647                labels: &[],
5648                origin: "github",
5649            })
5650            .await
5651            .unwrap();
5652        store
5653            .replace_dependencies(
5654                id,
5655                &[NewDependency {
5656                    repo: "owner/repo".into(),
5657                    number: 7,
5658                    source: DepSource::Github,
5659                }],
5660            )
5661            .await
5662            .unwrap();
5663        store
5664            .store_route(&StoreRoute {
5665                id,
5666                agent: "claude",
5667                model: Some("haiku"),
5668                complexity: "simple",
5669                estimate: 0,
5670                reason: "test",
5671                profile: "{}",
5672            })
5673            .await
5674            .unwrap();
5675        store.update_status(id, DbStatus::Routed).await.unwrap();
5676
5677        let dispatch_mode = DispatchMode {
5678            is_degraded: false,
5679            healthy_agents: 1,
5680            threshold: 1,
5681        };
5682        tick_dispatch_tasks(
5683            &backend,
5684            &tmux,
5685            "owner/repo",
5686            &runner,
5687            &capture,
5688            &semaphore,
5689            &task_manager,
5690            &weight_tx,
5691            dispatch_mode,
5692            &dispatching,
5693            &store,
5694            &HashMap::new(),
5695        )
5696        .await
5697        .unwrap();
5698
5699        let task = store.get(id).await.unwrap();
5700        assert_eq!(task.status, DbStatus::Waiting);
5701        assert!(dispatching.is_empty());
5702    }
5703
5704    /// Regression test for the completed-session -> stuck-task reclaim race.
5705    ///
5706    /// Scenario:
5707    /// - Runner finishes a tmux session and will call store_touch_updated_at shortly after
5708    /// - Engine's tick may observe the dead session in Phase 1 and Phase 2 may run
5709    ///   stuck-task recovery before the runner touches the store, causing incorrect reclaim
5710    /// Fix: tick_check_session_completions now touches the store.updated_at when it
5711    /// observes a completed session, preventing reclaim. This test simulates the
5712    /// timing by creating a session, marking the corresponding task InProgress with
5713    /// an old updated_at, calling tick_check_session_completions (which should touch
5714    /// the store), then calling tick_recover_stuck_tasks and asserting the task
5715    /// remains InProgress.
5716    #[tokio::test]
5717    async fn completed_session_does_not_cause_stuck_reclaim_race() {
5718        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5719        let mock = MockBackend::new();
5720        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5721        let task_manager = Arc::new(TaskManager::with_store(
5722            Arc::clone(&backend),
5723            Arc::clone(&store),
5724            "owner/repo".to_string(),
5725        ));
5726        let tmux = Arc::new(TmuxManager::new());
5727        let transport = Arc::new(Transport::new());
5728        let capture = Arc::new(CaptureService::new(transport));
5729
5730        // Insert external task and set to InProgress with old updated_at
5731        let id = store
5732            .upsert_external(&crate::store::UpsertExternal {
5733                repo: "owner/repo",
5734                ext_id: "200",
5735                title: "Race Task",
5736                body: "",
5737                author: "",
5738                url: "",
5739                labels: &[],
5740                origin: "github",
5741            })
5742            .await
5743            .unwrap();
5744        store
5745            .update_status(id, TaskStatus::InProgress)
5746            .await
5747            .unwrap();
5748
5749        // set updated_at to far past so it would normally be eligible for reclaim
5750        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5751            .bind(id)
5752            .execute(store.pool())
5753            .await
5754            .unwrap();
5755
5756        // Register a real tmux session for this task and then exit it so it's observed dead
5757        let task_id = "200";
5758        let session_name = tmux.session_name("owner/repo", task_id);
5759        let create_result = tmux
5760            .command()
5761            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5762            .output()
5763            .await;
5764        match create_result {
5765            Ok(o) if o.status.success() => {}
5766            _ => {
5767                eprintln!("Skipping test: tmux not available or failed to create test session");
5768                return;
5769            }
5770        }
5771        let _guard = SessionGuard(session_name.clone());
5772
5773        // Ensure remain-on-exit so the session lingers as dead long enough to be snapshot
5774        let _ = tmux
5775            .command()
5776            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5777            .output()
5778            .await;
5779        let _ = tmux
5780            .command()
5781            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5782            .output()
5783            .await;
5784
5785        // Poll until pane is dead
5786        for _ in 0..20 {
5787            time::sleep(Duration::from_millis(100)).await;
5788            if !tmux.session_is_running(&session_name).await {
5789                break;
5790            }
5791        }
5792
5793        // Register the session in capture (engine does this during dispatch)
5794        let task_key = task_id.to_string();
5795        capture
5796            .register_session("owner/repo", &task_key, &session_name)
5797            .await;
5798
5799        // Call phase1 (now touches store.updated_at for completed session)
5800        let config = EngineConfig {
5801            no_session_stuck_timeout: 600,
5802            stuck_timeout: 1800,
5803            ..EngineConfig::default()
5804        };
5805
5806        tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
5807            .await
5808            .unwrap();
5809
5810        // Verify phase1 actually touched updated_at. It only processes sessions
5811        // that appear in snapshot() (list_sessions + batch_session_active). On CI,
5812        // if the session was already fully removed before the snapshot runs (e.g.
5813        // remain-on-exit semantics differ across tmux versions), phase1 won't see
5814        // it and the store won't be touched. Skip in that case.
5815        {
5816            let task_state = store.get(id).await.unwrap();
5817            if task_state.updated_at.starts_with("2020") {
5818                eprintln!(
5819                    "Skipping test: tick_check_session_completions did not update stored \
5820                     updated_at — session was not in snapshot (CI tmux environment issue)"
5821                );
5822                return;
5823            }
5824        }
5825
5826        // Now run phase2 — the updated_at should have been touched so no reclaim occurs
5827        tick_recover_stuck_tasks(
5828            &backend,
5829            &tmux,
5830            "owner/repo",
5831            &task_manager,
5832            &config,
5833            &store,
5834            &HashMap::new(),
5835            &Arc::new(DashMap::new()),
5836        )
5837        .await
5838        .unwrap();
5839
5840        // Verify the task is still InProgress in the store
5841        let task = store.get(id).await.unwrap();
5842        assert_eq!(
5843            task.status,
5844            TaskStatus::InProgress,
5845            "task should not be reclaimed to New"
5846        );
5847    }
5848
5849    /// Regression test: a repo configured with a ".git" suffix (e.g. "owner/repo.git")
5850    /// must still match its own live tmux sessions in `tick_check_session_completions`.
5851    ///
5852    /// `TmuxManager::session_name` strips ".git" when creating the session, so a live
5853    /// session's parsed `project` field never carries it. Before the fix, this
5854    /// function's `repo_name` derivation skipped the `.trim_end_matches(".git")` step,
5855    /// so `session.project` ("repo") never equaled `repo_name` ("repo.git") and the
5856    /// completed-session touch/kill was silently skipped for the entire repo — leaving
5857    /// the coarser-grained stuck-task reclaim as the only (racy) safety net.
5858    #[tokio::test]
5859    async fn session_completion_matches_repo_configured_with_git_suffix() {
5860        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5861        let tmux = Arc::new(TmuxManager::new());
5862        let transport = Arc::new(Transport::new());
5863        let capture = Arc::new(CaptureService::new(transport));
5864
5865        let repo = "owner/repo.git";
5866        let id = store
5867            .upsert_external(&crate::store::UpsertExternal {
5868                repo,
5869                ext_id: "201",
5870                title: "Git-suffix repo task",
5871                body: "",
5872                author: "",
5873                url: "",
5874                labels: &[],
5875                origin: "github",
5876            })
5877            .await
5878            .unwrap();
5879        store
5880            .update_status(id, TaskStatus::InProgress)
5881            .await
5882            .unwrap();
5883        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5884            .bind(id)
5885            .execute(store.pool())
5886            .await
5887            .unwrap();
5888
5889        // session_name strips ".git" internally — this is the real session a dispatch
5890        // for this repo would have created.
5891        let task_id = "201";
5892        let session_name = tmux.session_name(repo, task_id);
5893        let create_result = tmux
5894            .command()
5895            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5896            .output()
5897            .await;
5898        match create_result {
5899            Ok(o) if o.status.success() => {}
5900            _ => {
5901                eprintln!("Skipping test: tmux not available or failed to create test session");
5902                return;
5903            }
5904        }
5905        let _guard = SessionGuard(session_name.clone());
5906
5907        let _ = tmux
5908            .command()
5909            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5910            .output()
5911            .await;
5912        let _ = tmux
5913            .command()
5914            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5915            .output()
5916            .await;
5917
5918        for _ in 0..20 {
5919            time::sleep(Duration::from_millis(100)).await;
5920            if !tmux.session_is_running(&session_name).await {
5921                break;
5922            }
5923        }
5924
5925        capture.register_session(repo, task_id, &session_name).await;
5926
5927        tick_check_session_completions(&tmux, repo, &capture, &store)
5928            .await
5929            .unwrap();
5930
5931        let task_state = store.get(id).await.unwrap();
5932        if task_state.updated_at.starts_with("2020") {
5933            eprintln!(
5934                "Skipping test: tick_check_session_completions did not update stored \
5935                 updated_at — session was not in snapshot (CI tmux environment issue)"
5936            );
5937            return;
5938        }
5939        assert!(
5940            !task_state.updated_at.starts_with("2020"),
5941            "tick_check_session_completions must match its own repo's live sessions \
5942             even when the repo is configured with a \".git\" suffix"
5943        );
5944    }
5945
5946    // ── tick_job_scheduler: resilience to bad project configs ────────────────
5947
5948    /// A project with a duplicate job id (same id in .orch.yml AND prompts/jobs/)
5949    /// must not cause tick_job_scheduler to return Err — it should log an error
5950    /// and return Ok(()) so other projects are unaffected.
5951    #[tokio::test]
5952    async fn job_scheduler_returns_ok_on_duplicate_job_config() {
5953        let tmp = tempfile::tempdir().unwrap();
5954        let cfg = tmp.path().join(".orch.yml");
5955        fs::write(
5956            &cfg,
5957            "jobs:\n  - id: trading-scan\n    schedule: \"0 9 * * *\"\n    task:\n      title: Scan\n      body: do it\n",
5958        )
5959        .unwrap();
5960        let jobs_dir = tmp.path().join("prompts").join("jobs");
5961        fs::create_dir_all(&jobs_dir).unwrap();
5962        fs::write(
5963            jobs_dir.join("trading-scan.md"),
5964            "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
5965        )
5966        .unwrap();
5967
5968        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5969        let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
5970        assert!(
5971            result.is_ok(),
5972            "duplicate job id must not propagate as Err: {result:?}"
5973        );
5974    }
5975
5976    /// When one project has a duplicate job id (bad config) and a second project
5977    /// has a valid config, the scheduler tick for the second project must still
5978    /// succeed — simulating the per-project loop in the engine.
5979    #[tokio::test]
5980    async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
5981        // Bad project: duplicate id across .orch.yml and prompts/jobs/
5982        let bad_tmp = tempfile::tempdir().unwrap();
5983        let bad_cfg = bad_tmp.path().join(".orch.yml");
5984        fs::write(
5985            &bad_cfg,
5986            "jobs:\n  - id: dup\n    schedule: \"0 9 * * *\"\n    task:\n      title: Dup\n      body: x\n",
5987        )
5988        .unwrap();
5989        let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
5990        fs::create_dir_all(&bad_jobs_dir).unwrap();
5991        fs::write(
5992            bad_jobs_dir.join("dup.md"),
5993            "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
5994        )
5995        .unwrap();
5996
5997        // Good project: valid config, no duplicate
5998        let good_tmp = tempfile::tempdir().unwrap();
5999        let good_cfg = good_tmp.path().join(".orch.yml");
6000        fs::write(
6001            &good_cfg,
6002            "jobs:\n  - id: valid-job\n    schedule: \"0 9 * * *\"\n    task:\n      title: Valid\n      body: ok\n",
6003        )
6004        .unwrap();
6005
6006        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
6007
6008        let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
6009        assert!(
6010            bad_result.is_ok(),
6011            "bad project must not abort scheduler: {bad_result:?}"
6012        );
6013
6014        let good_result =
6015            tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
6016        assert!(
6017            good_result.is_ok(),
6018            "valid project must still run after bad project: {good_result:?}"
6019        );
6020    }
6021
6022    // ── route_new_tasks_global ───────────────────────────────────────────────
6023
6024    /// Regression test for #3407: tasks from inactive/removed repos were stuck
6025    /// in `new` indefinitely because `tick_route_tasks` is repo-scoped.
6026    ///
6027    /// `route_new_tasks_global` must route tasks whose repo != current_repo
6028    /// and leave current-repo tasks untouched.
6029    #[serial_test::serial(cooldown_state)]
6030    #[tokio::test]
6031    async fn global_sweep_routes_inactive_repo_tasks() {
6032        cooldown::reset_global_state().await;
6033
6034        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6035
6036        // Active repo task — must NOT be touched by the global sweep.
6037        let active_id = store
6038            .upsert_external(&UpsertExternal {
6039                repo: "active/repo",
6040                ext_id: "1",
6041                title: "Active task",
6042                body: "",
6043                author: "user",
6044                url: "https://github.com/active/repo/issues/1",
6045                labels: &[],
6046                origin: "github",
6047            })
6048            .await
6049            .unwrap();
6050
6051        // Inactive repo task — must be routed by the global sweep.
6052        let inactive_id = store
6053            .upsert_external(&UpsertExternal {
6054                repo: "inactive/repo",
6055                ext_id: "42",
6056                title: "Orphaned task",
6057                body: "",
6058                author: "user",
6059                url: "https://github.com/inactive/repo/issues/42",
6060                labels: &[],
6061                origin: "github",
6062            })
6063            .await
6064            .unwrap();
6065
6066        assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
6067        assert_eq!(
6068            store.get(inactive_id).await.unwrap().status,
6069            TaskStatus::New
6070        );
6071
6072        let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6073        let mut tier = HashMap::new();
6074        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6075        model_map.insert("medium".to_string(), tier);
6076        let config = RouterConfig {
6077            mode: "round_robin".to_string(),
6078            agents: vec!["claude".to_string()],
6079            model_map,
6080            ..Default::default()
6081        };
6082        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6083
6084        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6085        let mut routing_quota = RoutingTickQuota::new(1);
6086        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6087            .await
6088            .unwrap();
6089
6090        // Active-repo task must remain new (global sweep skips it).
6091        assert_eq!(
6092            store.get(active_id).await.unwrap().status,
6093            TaskStatus::New,
6094            "global sweep must not touch tasks for the active repo"
6095        );
6096
6097        // Inactive-repo task must now be routed.
6098        let after = store.get(inactive_id).await.unwrap();
6099        assert_eq!(
6100            after.status,
6101            TaskStatus::Routed,
6102            "global sweep must route tasks from inactive repos"
6103        );
6104        assert!(
6105            matches!(&after.agent, Some(a) if !a.is_empty()),
6106            "agent must be set after routing"
6107        );
6108    }
6109
6110    /// Regression test for #3523: `route_new_tasks_global` previously filtered only
6111    /// `t.repo != current_repo`, so during repo A's tick it would route repo B's `new`
6112    /// tasks too — even though B is an active, non-orphaned project with its own tick
6113    /// scheduled later in the same pass. This is the exact "racing the real per-repo
6114    /// dispatch" failure mode `dispatch_routed_tasks_global` was fixed for in #3415;
6115    /// the route-side sweep needs the same `active_repos` scoping, not just the
6116    /// current tick's own repo.
6117    #[serial_test::serial(cooldown_state)]
6118    #[tokio::test]
6119    async fn global_sweep_does_not_touch_other_active_repo_tasks() {
6120        cooldown::reset_global_state().await;
6121
6122        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6123
6124        // Belongs to a second active repo — NOT the repo whose tick is running,
6125        // but still a currently-configured project, not an orphan.
6126        let other_active_id = store
6127            .upsert_external(&UpsertExternal {
6128                repo: "other-active/repo",
6129                ext_id: "7",
6130                title: "Other active project's task",
6131                body: "",
6132                author: "user",
6133                url: "https://github.com/other-active/repo/issues/7",
6134                labels: &[],
6135                origin: "github",
6136            })
6137            .await
6138            .unwrap();
6139
6140        // Genuinely orphaned repo — not in the active set at all.
6141        let inactive_id = store
6142            .upsert_external(&UpsertExternal {
6143                repo: "inactive/repo",
6144                ext_id: "42",
6145                title: "Orphaned task",
6146                body: "",
6147                author: "user",
6148                url: "https://github.com/inactive/repo/issues/42",
6149                labels: &[],
6150                origin: "github",
6151            })
6152            .await
6153            .unwrap();
6154
6155        let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6156        let mut tier = HashMap::new();
6157        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6158        model_map.insert("medium".to_string(), tier);
6159        let config = RouterConfig {
6160            mode: "round_robin".to_string(),
6161            agents: vec!["claude".to_string()],
6162            model_map,
6163            ..Default::default()
6164        };
6165        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6166
6167        // Simulate "active/repo"'s tick running the global sweep. Both
6168        // "active/repo" and "other-active/repo" are currently-configured
6169        // projects; only "inactive/repo" is orphaned.
6170        let active_repos: HashSet<String> =
6171            ["active/repo".to_string(), "other-active/repo".to_string()]
6172                .into_iter()
6173                .collect();
6174        let mut routing_quota = RoutingTickQuota::new(1);
6175        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6176            .await
6177            .unwrap();
6178
6179        assert_eq!(
6180            store.get(other_active_id).await.unwrap().status,
6181            TaskStatus::New,
6182            "global sweep run from a different active repo's tick must not route \
6183             another active repo's new tasks — that repo's own tick will route them"
6184        );
6185        assert_eq!(
6186            store.get(inactive_id).await.unwrap().status,
6187            TaskStatus::Routed,
6188            "global sweep must still route tasks from genuinely orphaned repos"
6189        );
6190    }
6191
6192    /// Tasks with the `no-agent` label must be skipped by the global sweep.
6193    #[serial_test::serial(cooldown_state)]
6194    #[tokio::test]
6195    async fn global_sweep_skips_no_agent_tasks() {
6196        cooldown::reset_global_state().await;
6197
6198        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6199
6200        let id = store
6201            .upsert_external(&UpsertExternal {
6202                repo: "inactive/repo",
6203                ext_id: "99",
6204                title: "No-agent task",
6205                body: "",
6206                author: "user",
6207                url: "https://github.com/inactive/repo/issues/99",
6208                labels: &["no-agent".to_string()],
6209                origin: "github",
6210            })
6211            .await
6212            .unwrap();
6213
6214        let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6215        let mut tier = HashMap::new();
6216        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6217        model_map.insert("medium".to_string(), tier);
6218        let config = RouterConfig {
6219            mode: "round_robin".to_string(),
6220            agents: vec!["claude".to_string()],
6221            model_map,
6222            ..Default::default()
6223        };
6224        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6225
6226        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6227        let mut routing_quota = RoutingTickQuota::new(1);
6228        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6229            .await
6230            .unwrap();
6231
6232        assert_eq!(
6233            store.get(id).await.unwrap().status,
6234            TaskStatus::New,
6235            "no-agent tasks must not be routed"
6236        );
6237    }
6238
6239    /// Regression test for #3615: the routing limit must be consumed by a
6240    /// shared per-engine-tick quota, not independently by every repo/global
6241    /// routing pass. Otherwise one slow router timeout per pass can stack into
6242    /// a multi-minute watchdog stall.
6243    #[serial_test::serial(cooldown_state)]
6244    #[tokio::test]
6245    async fn global_sweep_honors_shared_routing_quota() {
6246        cooldown::reset_global_state().await;
6247
6248        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6249        for ext_id in ["1", "2"] {
6250            store
6251                .upsert_external(&UpsertExternal {
6252                    repo: "inactive/repo",
6253                    ext_id,
6254                    title: "Orphaned task",
6255                    body: "",
6256                    author: "user",
6257                    url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
6258                    labels: &[],
6259                    origin: "github",
6260                })
6261                .await
6262                .unwrap();
6263        }
6264
6265        let mut model_map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
6266        let mut tier = HashMap::new();
6267        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
6268        model_map.insert("medium".to_string(), tier);
6269        let config = RouterConfig {
6270            mode: "round_robin".to_string(),
6271            agents: vec!["claude".to_string()],
6272            model_map,
6273            ..Default::default()
6274        };
6275        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
6276
6277        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6278        let mut routing_quota = RoutingTickQuota::new(1);
6279        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6280            .await
6281            .unwrap();
6282
6283        let all_tasks = store.list_all_global().await.unwrap();
6284        let routed = all_tasks
6285            .iter()
6286            .filter(|task| task.status == TaskStatus::Routed)
6287            .count();
6288        let still_new = all_tasks
6289            .iter()
6290            .filter(|task| task.status == TaskStatus::New)
6291            .count();
6292
6293        assert_eq!(routed, 1, "only one task may consume the shared quota");
6294        assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
6295
6296        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
6297            .await
6298            .unwrap();
6299        let after_exhausted = store.list_all_global().await.unwrap();
6300        assert_eq!(
6301            after_exhausted
6302                .iter()
6303                .filter(|task| task.status == TaskStatus::Routed)
6304                .count(),
6305            1,
6306            "an exhausted shared quota must not reset within the same tick"
6307        );
6308    }
6309
6310    /// A task of a repo with no project directory ends `blocked` with a reason that names
6311    /// the cause, never `needs_review` (#4024)
6312    #[tokio::test]
6313    async fn block_task_without_project_blocks_with_the_cause() {
6314        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6315        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
6316        let task_manager = Arc::new(TaskManager::with_store(
6317            backend,
6318            Arc::clone(&store),
6319            "removed/repo".to_string(),
6320        ));
6321        let id = store
6322            .create_internal("removed/repo", "Orphan", "", "cron", "1", None)
6323            .await
6324            .unwrap();
6325        store.update_status(id, TaskStatus::Routed).await.unwrap();
6326
6327        block_task_without_project(
6328            &task_manager,
6329            &store,
6330            "removed/repo",
6331            &format!("internal:{id}"),
6332        )
6333        .await;
6334
6335        let task = store.get(id).await.unwrap();
6336        assert_eq!(task.status, TaskStatus::Blocked);
6337        assert_eq!(
6338            task.block_reason.as_deref(),
6339            Some(NoProjectDir::BLOCK_REASON)
6340        );
6341    }
6342
6343    // ── dispatch_routed_tasks_global ─────────────────────────────────────────
6344    //
6345    // `dispatch_routed_tasks_global` itself is deliberately not exercised
6346    // end-to-end here: once it finds an orphaned repo, it spawns the real
6347    // runner, which creates a real tmux session and touches the network.
6348    // The existing `dispatch_does_not_deadlock_under_write_lock` test avoids
6349    // this same hazard by keeping the dispatchable set empty. The selection
6350    // logic (which tasks belong to which repo) is pulled into the pure
6351    // `group_orphaned_routed_tasks` helper below so it can be tested without
6352    // those side effects.
6353
6354    /// Regression test for #3413: tasks routed for an inactive/removed repo were
6355    /// stuck in `routed` indefinitely because `tick_dispatch_tasks` is repo-scoped.
6356    ///
6357    /// `group_orphaned_routed_tasks` must select tasks whose repo != the active
6358    /// repo, group them by repo, and leave active-repo tasks out entirely.
6359    #[tokio::test]
6360    async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
6361        async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6362            let id = store
6363                .upsert_external(&UpsertExternal {
6364                    repo,
6365                    ext_id,
6366                    title: "Routed task",
6367                    body: "",
6368                    author: "user",
6369                    url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6370                    labels: &[],
6371                    origin: "github",
6372                })
6373                .await
6374                .unwrap();
6375            store
6376                .store_route(&StoreRoute {
6377                    id,
6378                    agent: "claude",
6379                    model: Some("sonnet"),
6380                    complexity: "medium",
6381                    estimate: 0,
6382                    reason: "test",
6383                    profile: "{}",
6384                })
6385                .await
6386                .unwrap();
6387            store.update_status(id, TaskStatus::Routed).await.unwrap();
6388            id
6389        }
6390
6391        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6392
6393        let active_id = make_routed(&store, "active/repo", "1").await;
6394        let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
6395        let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
6396        let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
6397
6398        let all_routed = store
6399            .list_all_by_status_global(TaskStatus::Routed)
6400            .await
6401            .unwrap();
6402        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6403        let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6404
6405        assert!(
6406            !by_repo.contains_key("active/repo"),
6407            "global sweep must not select tasks for the active repo"
6408        );
6409        let inactive_ids: HashSet<i64> = by_repo
6410            .get("inactive/repo")
6411            .expect("inactive/repo must be selected")
6412            .iter()
6413            .map(|t| t.id)
6414            .collect();
6415        assert_eq!(
6416            inactive_ids,
6417            [inactive_id_a, inactive_id_b].into_iter().collect(),
6418            "global sweep must group all orphaned tasks for a repo together"
6419        );
6420        assert_eq!(
6421            by_repo
6422                .get("other-inactive/repo")
6423                .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6424            Some(vec![other_inactive_id])
6425        );
6426
6427        // Sanity: the active-repo task really was created and routed, it's just
6428        // excluded from the selection above.
6429        assert_eq!(
6430            store.get(active_id).await.unwrap().status,
6431            TaskStatus::Routed
6432        );
6433    }
6434
6435    /// Regression test for the PR #3413 review finding: a single `current_repo` string is
6436    /// the wrong exclusion — repo B's routed tasks must not look "orphaned" during repo A's
6437    /// dispatch just because A != B.
6438    /// `group_orphaned_routed_tasks` must exclude every repo in the full active set.
6439    #[tokio::test]
6440    async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
6441        async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6442            let id = store
6443                .upsert_external(&UpsertExternal {
6444                    repo,
6445                    ext_id,
6446                    title: "Routed task",
6447                    body: "",
6448                    author: "user",
6449                    url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6450                    labels: &[],
6451                    origin: "github",
6452                })
6453                .await
6454                .unwrap();
6455            store
6456                .store_route(&StoreRoute {
6457                    id,
6458                    agent: "claude",
6459                    model: Some("sonnet"),
6460                    complexity: "medium",
6461                    estimate: 0,
6462                    reason: "test",
6463                    profile: "{}",
6464                })
6465                .await
6466                .unwrap();
6467            store.update_status(id, TaskStatus::Routed).await.unwrap();
6468            id
6469        }
6470
6471        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6472
6473        let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
6474        let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
6475        let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
6476
6477        let all_routed = store
6478            .list_all_by_status_global(TaskStatus::Routed)
6479            .await
6480            .unwrap();
6481        // Simulate repo A's tick: current repo is "active/repo-a", but the full
6482        // active project set also includes "active/repo-b".
6483        let active_repos: HashSet<String> =
6484            ["active/repo-a".to_string(), "active/repo-b".to_string()]
6485                .into_iter()
6486                .collect();
6487        let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6488
6489        assert!(
6490            !by_repo.contains_key("active/repo-a"),
6491            "the current repo's own tasks must not be swept"
6492        );
6493        assert!(
6494            !by_repo.contains_key("active/repo-b"),
6495            "another active repo's tasks must not be swept during repo A's tick"
6496        );
6497        assert_eq!(
6498            by_repo
6499                .get("inactive/repo")
6500                .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6501            Some(vec![orphaned_id]),
6502            "a genuinely inactive repo's tasks must still be swept"
6503        );
6504
6505        // Sanity: repo A and repo B tasks were really created and routed, just excluded.
6506        assert_eq!(
6507            store.get(repo_a_id).await.unwrap().status,
6508            TaskStatus::Routed
6509        );
6510        assert_eq!(
6511            store.get(repo_b_id).await.unwrap().status,
6512            TaskStatus::Routed
6513        );
6514    }
6515
6516    /// Tasks with the `no-agent` label must be skipped by the global dispatch sweep.
6517    #[tokio::test]
6518    async fn global_dispatch_sweep_skips_no_agent_tasks() {
6519        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6520
6521        let id = store
6522            .upsert_external(&UpsertExternal {
6523                repo: "inactive/repo",
6524                ext_id: "99",
6525                title: "No-agent routed task",
6526                body: "",
6527                author: "user",
6528                url: "https://github.com/inactive/repo/issues/99",
6529                labels: &["no-agent".to_string()],
6530                origin: "github",
6531            })
6532            .await
6533            .unwrap();
6534        store
6535            .store_route(&StoreRoute {
6536                id,
6537                agent: "claude",
6538                model: Some("sonnet"),
6539                complexity: "medium",
6540                estimate: 0,
6541                reason: "test",
6542                profile: "{}",
6543            })
6544            .await
6545            .unwrap();
6546        store.update_status(id, TaskStatus::Routed).await.unwrap();
6547
6548        // Safe to call the full sweep here: with the only orphaned task filtered
6549        // out by the no-agent label, `group_orphaned_routed_tasks` returns an
6550        // empty map and `dispatch_routed_tasks_global` returns before ever
6551        // constructing a backend or spawning a runner.
6552        let tmux = Arc::new(TmuxManager::new());
6553        let transport = Arc::new(Transport::new());
6554        let capture = Arc::new(CaptureService::new(transport));
6555        let semaphore = Arc::new(Semaphore::new(4));
6556        let (weight_tx, _weight_rx) = mpsc::channel(16);
6557        let dispatching = Arc::new(DashMap::new());
6558        let dispatch_mode = DispatchMode {
6559            is_degraded: false,
6560            healthy_agents: 1,
6561            threshold: 1,
6562        };
6563
6564        let active_repos: HashSet<String> = ["active/repo".to_string()].into_iter().collect();
6565        dispatch_routed_tasks_global(
6566            &tmux,
6567            &capture,
6568            &semaphore,
6569            &weight_tx,
6570            dispatch_mode,
6571            &dispatching,
6572            &store,
6573            &HashMap::new(),
6574            &active_repos,
6575        )
6576        .await
6577        .unwrap();
6578
6579        assert_eq!(
6580            store.get(id).await.unwrap().status,
6581            TaskStatus::Routed,
6582            "no-agent tasks must not be dispatched"
6583        );
6584    }
6585}