Skip to main content

orch/engine/
tick.rs

1//! Core tick loop phases.
2//!
3//! The engine ticks every ~10 seconds. Each tick runs six phases:
4//! 1. Poll tmux for finished sessions
5//! 2. Recover stuck in-progress tasks
6//! 3. Route and dispatch:
7//!    - 3a. Route new tasks to agents
8//!    - 3b. Dispatch routed tasks (spawn agents)
9//! 4. Unblock parents whose children are all done
10//! 5. Run cron job scheduler
11//!
12//! [`tick_all`] runs them for every project. Phases 1-2 and 3b-5 run for all projects at
13//! once; only routing is serial, because it holds the router write lock. Store-wide work
14//! (`tick_global_sweeps`, `reconcile_dependencies`) runs once per tick, not once per project.
15
16use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
17use crate::channels::capture::CaptureService;
18use crate::config;
19use crate::engine::cooldown::{
20    SILENCE_AGENT_COOLDOWN_SECS, SILENCE_EXTENDED_COOLDOWN_SECS, github_circuit_remaining_secs,
21    is_agent_in_cooldown, is_github_circuit_open, is_model_in_cooldown,
22    record_agent_failure_with_message, record_silence_detection, set_agent_cooldown,
23    set_model_cooldown,
24};
25use crate::engine::dispatch_guard::DispatchGuard;
26use crate::engine::jobs;
27use crate::engine::router::{AllAgentsCooledError, RouteResultError, Router, get_route_result};
28use crate::engine::runner::{TaskRunner, WeightSignal};
29use crate::engine::tasks::{TaskManager, is_internal_id};
30use crate::repo_context::REPO_CONTEXT;
31use crate::store::TaskStore;
32use crate::tmux::TmuxManager;
33use dashmap::{DashMap, DashSet};
34use std::sync::Arc;
35use std::time::{Instant, SystemTime};
36use tokio::sync::{Semaphore, mpsc};
37
38/// Per outer engine-loop quota for expensive routing calls.
39///
40/// Routing runs once per active project. Without a quota shared by all of them,
41/// `router.max_tasks_per_tick = 1` still permits one slow LLM routing call per
42/// project, multiplying 45s router timeouts into multi-minute watchdog stalls.
43pub(crate) struct RoutingTickQuota {
44    remaining: usize,
45}
46
47impl RoutingTickQuota {
48    pub(crate) fn new(max: usize) -> Self {
49        Self { remaining: max }
50    }
51
52    fn is_exhausted(&self) -> bool {
53        self.remaining == 0
54    }
55
56    fn remaining(&self) -> usize {
57        self.remaining
58    }
59
60    fn consume_one(&mut self) -> bool {
61        if self.remaining == 0 {
62            return false;
63        }
64        self.remaining -= 1;
65        true
66    }
67}
68
69/// Cleanup stale tmux sessions from previous engine runs.
70/// Called once from `serve()` before the main loop, never from a tick
71pub(crate) async fn startup_cleanup(tmux: &TmuxManager) {
72    let start_time = SystemTime::now()
73        .duration_since(SystemTime::UNIX_EPOCH)
74        .unwrap_or_default()
75        .as_secs();
76    tracing::info!(
77        start_time,
78        "running startup cleanup for stale tmux sessions"
79    );
80
81    match tmux.kill_stale_sessions(start_time).await {
82        Ok(killed) => {
83            if killed > 0 {
84                tracing::info!(killed, "startup cleanup killed stale sessions");
85            } else {
86                tracing::debug!("no stale tmux sessions found on startup");
87            }
88        }
89        Err(e) => {
90            tracing::warn!(error = %e, "failed to run startup cleanup");
91        }
92    }
93}
94
95use super::{EngineConfig, ProjectEngine};
96use crate::store::{store_set_by_id, store_touch_updated_at};
97
98async fn recover_routed_blocked_dispatch(
99    backend: &Arc<dyn ExternalBackend>,
100    store: &Arc<TaskStore>,
101    task_manager: &Arc<TaskManager>,
102    repo: &str,
103    task: &ExternalTask,
104    timing: &StuckTaskTiming,
105) {
106    tracing::warn!(
107        task_id = task.id.0,
108        age_mins = timing.age.num_minutes(),
109        threshold_mins = timing.threshold / 60,
110        "recovering routed task: dispatch blocked by stale tmux session → new"
111    );
112
113    let resolved_store_id = match store.resolve_task_id(repo, &task.id.0).await {
114        Ok(Some(id)) => Some(id),
115        Ok(None) => {
116            tracing::warn!(
117                task_id = %task.id.0,
118                repo,
119                "resolve_task_id returned None during routed-task recovery"
120            );
121            None
122        }
123        Err(e) => {
124            tracing::error!(
125                task_id = %task.id.0,
126                repo,
127                error = %e,
128                "resolve_task_id failed during routed-task recovery"
129            );
130            None
131        }
132    };
133
134    if let Some(store_id) = resolved_store_id {
135        if let Err(e) = store_set_by_id(
136            Some(store),
137            store_id,
138            &[
139                ("agent", serde_json::Value::Null),
140                ("model", serde_json::Value::Null),
141                ("route_attempts", serde_json::json!(0)),
142            ],
143        )
144        .await
145        {
146            tracing::warn!(
147                task_id = task.id.0,
148                ?e,
149                "failed to clear routing fields for routed stuck task — will retry next tick"
150            );
151            return;
152        }
153        finalize_open_runs_for_recovery(
154            store,
155            store_id,
156            "stuck-task recovery: stale session blocking dispatch killed",
157        )
158        .await;
159    }
160
161    if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
162        tracing::warn!(
163            task_id = task.id.0,
164            ?e,
165            "failed to reset routed stuck task status"
166        );
167        return;
168    }
169
170    crate::store::store_log_activity(
171        &Some(Arc::clone(store)),
172        repo,
173        &task.id.0,
174        "rerouted",
175        Some("routed"),
176        Some("new"),
177        None::<&str>,
178        None::<&str>,
179        Some(&serde_json::json!({
180            "failure_reason": "routed_blocked_by_existing_session",
181            "age_minutes": timing.age.num_minutes(),
182            "had_session": true,
183        })),
184    )
185    .await;
186
187    if !is_internal_id(&task.id.0) {
188        let backend_clone = Arc::clone(backend);
189        let task_id_clone = task.id.clone();
190        let body = format!(
191            "[{}] recovered: stuck routed task — existing tmux session blocked dispatch for {}m; cleared route for retry{}",
192            chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
193            timing.age.num_minutes(),
194            crate::engine::orch_footer()
195        );
196        tokio::spawn(async move {
197            if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
198                tracing::warn!(
199                    task_id = task_id_clone.0,
200                    ?e,
201                    "failed to post routed-task recovery comment"
202                );
203            }
204        });
205    }
206}
207
208/// Phase 1 of tick: poll tmux for finished sessions and clean them up.
209pub(crate) async fn tick_check_session_completions(
210    tmux: &Arc<TmuxManager>,
211    repo: &str,
212    capture: &Arc<CaptureService>,
213    store: &Arc<TaskStore>,
214) -> anyhow::Result<()> {
215    let _span = tracing::info_span!("engine.tick.phase1.sessions").entered();
216    let session_snapshot = tmux.snapshot().await;
217    // Derive the short project name from repo (owner/repo -> repo), matching
218    // TmuxManager::session_name's derivation exactly (including the ".git" strip)
219    // so a repo configured with a ".git" suffix doesn't silently fail to match
220    // its own live sessions here — leaving stuck-task recovery as the only
221    // (later, coarser-grained) safety net for this repo's session completions.
222    let repo_name = repo
223        .rsplit('/')
224        .next()
225        .unwrap_or(repo)
226        .trim_end_matches(".git");
227    for (session, active) in session_snapshot {
228        // Only handle sessions that belong to this repo/project.
229        if session.project != repo_name {
230            continue;
231        }
232
233        if !active {
234            tracing::info!(
235                session = %session.name,
236                task_id = %session.task_id,
237                "session completed, collecting results"
238            );
239            // Touch the store.updated_at immediately when we observe a completed
240            // session so the stuck-task recovery phase doesn't incorrectly
241            // reclaim this task while the runner is still finishing
242            // post-processing (race: session observed dead → tick reclaim).
243            // Best-effort: no-ops if the task isn't present in the store.
244            //
245            // session.task_id uses hyphens (e.g. "internal-63714") because the
246            // tmux session name sanitizes colons to hyphens. resolve_task_id only
247            // handles "internal:" prefix, so we must convert back before touching.
248            // Review sessions carry a "-review" suffix tracked under the main task.
249            let store_task_id = {
250                let base = session
251                    .task_id
252                    .strip_suffix("-review")
253                    .unwrap_or(&session.task_id);
254                if let Some(n) = base.strip_prefix("internal-") {
255                    format!("internal:{n}")
256                } else {
257                    base.to_string()
258                }
259            };
260            store_touch_updated_at(&Some(Arc::clone(store)), repo, &store_task_id).await;
261            // Unregister from capture service using the task id the capture service
262            // was registered under (task id without project prefix).
263            capture.unregister_session(repo, &session.task_id).await;
264            // Kill the actual tmux session name we discovered (do not reconstruct)
265            if let Err(e) = tmux.kill_session(&session.name).await {
266                tracing::debug!(
267                    session = %session.name,
268                    ?e,
269                    "kill_session failed (session may already be gone)"
270                );
271            }
272        }
273    }
274    Ok(())
275}
276
277#[derive(Debug)]
278struct StuckTaskTiming {
279    has_session: bool,
280    age: chrono::Duration,
281    threshold: u64,
282}
283
284fn stuck_task_timing_from_map(
285    tmux: &Arc<TmuxManager>,
286    repo: &str,
287    session_task_id: &str,
288    updated_at: &str,
289    config: &EngineConfig,
290    parse_error_message: &'static str,
291    session_map: &std::collections::HashMap<String, bool>,
292) -> Option<StuckTaskTiming> {
293    let session_name = tmux.session_name(repo, session_task_id);
294    // A session is "running" if it appears in the map with alive=true.
295    let has_session = session_map.get(&session_name).copied().unwrap_or(false);
296    let threshold = if has_session {
297        config.stuck_timeout
298    } else {
299        config.no_session_stuck_timeout
300    };
301
302    let updated = match chrono::DateTime::parse_from_rfc3339(updated_at) {
303        Ok(dt) => dt.with_timezone(&chrono::Utc),
304        Err(e) => {
305            tracing::warn!(
306                task_id = session_task_id,
307                updated_at,
308                ?e,
309                error_message = parse_error_message,
310                "cannot parse updated_at"
311            );
312            return None;
313        }
314    };
315
316    let raw_age = chrono::Utc::now() - updated;
317    // Host suspend (laptop sleep) freezes the process just like it freezes the agent
318    // running inside it — that frozen time is not evidence of a hang. Discount any
319    // detected suspend/resume gaps that occurred since this task was last touched.
320    let suspended = super::suspend::suspended_duration_since(updated);
321    let age = (raw_age - suspended).max(chrono::Duration::zero());
322    if age.num_seconds() <= i64::try_from(threshold).unwrap_or(i64::MAX) {
323        return None;
324    }
325
326    Some(StuckTaskTiming {
327        has_session,
328        age,
329        threshold,
330    })
331}
332
333/// When the task last entered in_review, and whether a review decision landed since. Age from the
334/// status entry: `review_open_prs` rewrites `updated_at` on every pass, so it never ages
335async fn in_review_stuck_entry(store: &TaskStore, store_id: i64, fallback: &str) -> (String, bool) {
336    match store.in_review_entry(store_id).await {
337        Ok(Some(entry)) => entry,
338        Ok(None) => (fallback.to_string(), false),
339        Err(e) => {
340            tracing::warn!(
341                store_id,
342                error = %e,
343                "failed to read in_review entry, falling back to updated_at"
344            );
345            (fallback.to_string(), false)
346        }
347    }
348}
349
350/// Finalize any open `task_runs` rows for a task during recovery paths.
351/// Best-effort: logs a warning on failure but does not block recovery.
352async fn finalize_open_runs_for_recovery(store: &TaskStore, store_id: i64, reason: &str) {
353    if let Err(e) = store
354        .finalize_incomplete_runs(store_id, "aborted", reason)
355        .await
356    {
357        tracing::warn!(
358            store_id,
359            error = %e,
360            reason,
361            "failed to finalize incomplete runs during recovery"
362        );
363    }
364}
365
366/// Phase 1b of tick: detect agents that have produced no output since session start
367/// and have exceeded the silence grace period.
368///
369/// Only triggers on complete silence (agent never produced any output). Agents that
370/// produced output then went quiet (e.g. long tool calls) are NOT affected.
371///
372/// On silence: kills the tmux session, applies model-level cooldown, resets the task
373/// to `New` so the router picks a different agent/model.
374pub(crate) async fn tick_detect_silent_agents(
375    tmux: &Arc<TmuxManager>,
376    repo: &str,
377    capture: &Arc<CaptureService>,
378    backend: &Arc<dyn ExternalBackend>,
379    task_manager: &Arc<TaskManager>,
380    config: &EngineConfig,
381    store: &Arc<TaskStore>,
382) -> anyhow::Result<()> {
383    let _span = tracing::info_span!("engine.tick.phase1b.silence").entered();
384    let grace = std::time::Duration::from_secs(config.silence_grace_period);
385    let silent_sessions = capture.get_silent_sessions_for_repo(repo, grace).await;
386
387    for (task_id, session_name, silence_age_secs) in silent_sessions {
388        let use_backend = should_use_backend(&task_id);
389        // Look up agent + model from the store so we can cooldown the right model.
390        let store_task = match store.resolve_task_id(repo, &task_id).await {
391            Ok(Some(store_id)) => match store.get(store_id).await {
392                Ok(task) => Some(task),
393                Err(e) => {
394                    tracing::warn!(
395                        task_id,
396                        repo,
397                        store_id,
398                        operation = "store.get",
399                        error = %e,
400                        "silence detection: db error fetching task — agent/model cooldown will be skipped"
401                    );
402                    None
403                }
404            },
405            Ok(None) => {
406                tracing::debug!(
407                    task_id,
408                    repo,
409                    operation = "resolve_task_id",
410                    "silence detection: task not found in store (may have been cleaned up)"
411                );
412                None
413            }
414            Err(e) => {
415                tracing::warn!(
416                    task_id,
417                    repo,
418                    operation = "resolve_task_id",
419                    error = %e,
420                    "silence detection: db error resolving task id — agent/model cooldown will be skipped"
421                );
422                None
423            }
424        };
425        let agent_name = store_task
426            .as_ref()
427            .and_then(|t| t.agent.clone())
428            .unwrap_or_default();
429        let model_name = store_task
430            .as_ref()
431            .and_then(|t| t.model.clone())
432            .unwrap_or_default();
433
434        tracing::warn!(
435            task_id,
436            agent = %agent_name,
437            model = %model_name,
438            silence_age_secs,
439            grace_secs = config.silence_grace_period,
440            cooldown_secs = config.silence_cooldown,
441            "agent silent since session start — killing session, cooling down model + agent, failing over"
442        );
443
444        // 1. Kill the tmux session
445        if let Err(e) = tmux.kill_session(&session_name).await {
446            tracing::debug!(
447                task_id,
448                ?e,
449                "kill_session failed for silent agent (may already be gone)"
450            );
451        }
452
453        // 2. Unregister from capture
454        capture.unregister_session(repo, &task_id).await;
455
456        // Finalize any open audit run so the task_runs row is not orphaned
457        if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
458            finalize_open_runs_for_recovery(store, store_id, "silence detection: session killed")
459                .await;
460        }
461
462        let mut extended_note = String::new();
463
464        // 3. Cooldown the specific model (not the whole agent)
465        if !agent_name.is_empty() && !model_name.is_empty() {
466            set_model_cooldown(
467                &agent_name,
468                &model_name,
469                config.silence_cooldown,
470                "silence_detected",
471            )
472            .await;
473            if let Some(result) = record_silence_detection(&agent_name, &model_name).await
474                && result.extended_cooldown_applied
475            {
476                extended_note = format!(
477                    " ({} silences in 24h -> extended cooldown {}s)",
478                    result.count, SILENCE_EXTENDED_COOLDOWN_SECS
479                );
480            }
481        }
482
483        // 3b. Short agent-level cooldown to force router to pick a different agent.
484        // Without this, the router picks the same agent with a different model,
485        // looping through all models (~2 min each) before the long agent cooldown kicks in.
486        if !agent_name.is_empty() {
487            set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
488        }
489
490        // 4. Pick a fallback agent and set to Routed (not New) to preserve progress.
491        // Setting to New would clear routing state and trigger a full LLM re-routing
492        // cycle, losing intermediate context. Routed skips re-routing and re-dispatches
493        // directly with the chosen fallback agent.
494        let task_eid = ExternalId(task_id.clone());
495
496        // Build available agents list and reroute chain for failover
497        let available: Vec<String> = crate::engine::configured_agents()
498            .into_iter()
499            .filter(|a| crate::cmd_cache::command_exists(a))
500            .collect();
501        let chain = crate::engine::runner::response::get_reroute_chain(
502            &task_id,
503            &Some(Arc::clone(store)),
504            repo,
505        )
506        .await;
507        let chain = crate::engine::runner::response::update_reroute_chain(
508            &task_id,
509            &agent_name,
510            &chain,
511            &Some(Arc::clone(store)),
512            repo,
513        )
514        .await;
515
516        let (next_status, next_agent) = if let Some(fallback) =
517            crate::engine::runner::response::pick_fallback_agent(&agent_name, &chain, &available)
518        {
519            tracing::info!(
520                task_id,
521                from = %agent_name,
522                to = %fallback,
523                "silence detection: failover to different agent, setting routed"
524            );
525            (Status::Routed, Some(fallback))
526        } else {
527            tracing::warn!(
528                task_id,
529                "silence detection: no fallback agents available, resetting to new for re-routing"
530            );
531            (Status::New, None)
532        };
533
534        if use_backend && let Some(ref st) = store_task {
535            // Fire-and-forget: label removals are cosmetic — tick should not block on them.
536            let stale_labels: Vec<String> = st
537                .labels
538                .iter()
539                .filter(|l| {
540                    l.starts_with("agent:")
541                        || l.starts_with("complexity:")
542                        || l.starts_with("model:")
543                })
544                .cloned()
545                .collect();
546            if !stale_labels.is_empty() {
547                let backend_clone = Arc::clone(backend);
548                let task_eid_clone = task_eid.clone();
549                tokio::spawn(async move {
550                    for label in &stale_labels {
551                        if let Err(e) = backend_clone.remove_label(&task_eid_clone, label).await {
552                            tracing::warn!(task_id = task_eid_clone.0, label, error = %e, "failed to remove label during silence detection re-route");
553                        }
554                    }
555                });
556            }
557        }
558
559        if let Some(store_id) = store_task.as_ref().map(|t| t.id) {
560            if let Some(ref fallback) = next_agent {
561                // Write the fallback agent into the store so dispatch can proceed
562                // directly without an LLM re-routing cycle. Clearing model forces
563                // model_for_complexity to pick the best available model for the new agent.
564                let _ = store_set_by_id(
565                    Some(store),
566                    store_id,
567                    &[
568                        ("agent", serde_json::json!(fallback)),
569                        ("model", serde_json::json!("")),
570                        (
571                            "last_error",
572                            serde_json::json!(format!(
573                                "silence detected after {}s, failing over to {}",
574                                config.silence_grace_period, fallback
575                            )),
576                        ),
577                    ],
578                )
579                .await;
580            } else {
581                let _ = store_set_by_id(
582                    Some(store),
583                    store_id,
584                    &[
585                        ("agent", serde_json::Value::Null),
586                        ("model", serde_json::Value::Null),
587                        (
588                            "last_error",
589                            serde_json::json!(format!(
590                                "silence detected after {}s, no fallback agents available",
591                                config.silence_grace_period
592                            )),
593                        ),
594                    ],
595                )
596                .await;
597            }
598        }
599
600        if let Err(e) = task_manager
601            .update_task_status(&task_eid, next_status)
602            .await
603        {
604            tracing::warn!(task_id, ?e, "failed to update silent task status");
605            continue;
606        }
607
608        // Log reroute activity
609        let details = serde_json::json!({
610            "failure_reason": "agent_silence",
611            "silence_duration_secs": config.silence_grace_period,
612            "cooldown_applied": !agent_name.is_empty() && !model_name.is_empty(),
613            "fallback_available": next_agent.is_some(),
614        });
615        crate::store::store_log_activity(
616            &Some(Arc::clone(store)),
617            repo,
618            &task_id,
619            "rerouted",
620            Some("in_progress"),
621            Some(next_status.as_str()),
622            next_agent.as_deref(),
623            None,
624            Some(&details),
625        )
626        .await;
627
628        // 5. Post a comment explaining what happened
629        let action = if let Some(ref fallback) = next_agent {
630            format!("failing over to {fallback}")
631        } else {
632            "marking needs_review (no fallback agents)".to_string()
633        };
634        let comment = format!(
635            "[{}] agent silent for {}s since session start — killed session, cooled down model `{}:{}` for {}s{}, agent `{}` for {}s, {}{}",
636            chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
637            config.silence_grace_period,
638            agent_name,
639            model_name,
640            config.silence_cooldown,
641            extended_note,
642            agent_name,
643            SILENCE_AGENT_COOLDOWN_SECS,
644            action,
645            crate::engine::orch_footer(),
646        );
647        if use_backend {
648            // Fire-and-forget: comment is informational — tick should not block on it.
649            let backend_clone = Arc::clone(backend);
650            let task_eid_clone = task_eid.clone();
651            tokio::spawn(async move {
652                if let Err(e) = backend_clone.post_comment(&task_eid_clone, &comment).await {
653                    tracing::warn!(
654                        task_id = task_eid_clone.0,
655                        ?e,
656                        "failed to post silence detection comment"
657                    );
658                }
659            });
660        }
661    }
662
663    Ok(())
664}
665
666fn should_use_backend(task_id: &str) -> bool {
667    !is_internal_id(task_id)
668}
669
670/// Phase 2 of tick: recover tasks stranded by missing or stale tmux state.
671/// - `in_progress` tasks → reset to `New` when their session is missing or the run timed out
672/// - `routed` tasks      → reset to `New` when an old live session keeps blocking re-dispatch
673/// - `in_review` tasks   → reset to `NeedsReview` when the review session disappeared
674#[expect(
675    clippy::too_many_arguments,
676    reason = "each argument is engine state the tick passes down"
677)]
678pub(crate) async fn tick_recover_stuck_tasks(
679    backend: &Arc<dyn ExternalBackend>,
680    tmux: &Arc<TmuxManager>,
681    repo: &str,
682    task_manager: &Arc<TaskManager>,
683    config: &EngineConfig,
684    store: &Arc<TaskStore>,
685    session_map: &std::collections::HashMap<String, bool>,
686    dispatching: &Arc<DashMap<String, String>>,
687) -> anyhow::Result<()> {
688    use crate::store::TaskStatus as DbStatus;
689
690    let _span = tracing::info_span!("engine.tick.phase2.stuck_tasks").entered();
691
692    let in_progress = match task_manager
693        .list_external_by_status(Status::InProgress)
694        .await
695    {
696        Ok(tasks) => tasks,
697        Err(e) => {
698            tracing::warn!(?e, "failed to list in_progress tasks for stuck recovery");
699            vec![]
700        }
701    };
702    for task in &in_progress {
703        let Some(timing) = stuck_task_timing_from_map(
704            tmux,
705            repo,
706            &task.id.0,
707            &task.updated_at,
708            config,
709            "cannot parse updated_at, skipping stuck-task check",
710            session_map,
711        ) else {
712            continue;
713        };
714
715        // A dead tmux session with no `updated_at` refresh in a long time normally means
716        // the runner vanished (crashed, orch restarted). But it also matches the tail end
717        // of a *successful* run: the session exits the instant the agent finishes, while
718        // the runner is still polling for that exit (up to its own poll interval) and then
719        // writing results. If the runner's dispatch guard for this task is still held, the
720        // runner is alive and actively processing — let it finish rather than racing it.
721        if !timing.has_session {
722            let dispatch_key = format!("{repo}/{}", task.id.0);
723            if dispatching.contains_key(&dispatch_key) {
724                // Promoted from debug! to info!: the default log filter is "orch=info"
725                // (see main.rs), so this line — the only direct evidence that the guard
726                // check actually found and honored a held dispatch guard — was silently
727                // dropped in production. Every prior occurrence of the reclaim race
728                // (#3518/#3519/#3523/#3525/#3526) had to be diagnosed from timestamp
729                // correlation alone because this and the "dispatch guard
730                // claimed"/"released" lines never appeared in orch.log.
731                tracing::info!(
732                    task_id = task.id.0,
733                    dispatch_key,
734                    age_mins = timing.age.num_minutes(),
735                    "skipping stuck-task reclaim: runner still actively processing this task"
736                );
737                continue;
738            }
739        }
740
741        if timing.has_session {
742            tracing::warn!(
743                task_id = task.id.0,
744                age_mins = timing.age.num_minutes(),
745                threshold_mins = timing.threshold / 60,
746                "recovering stuck task: timed out with active session → new"
747            );
748        } else {
749            tracing::warn!(
750                task_id = task.id.0,
751                dispatch_key = format!("{repo}/{}", task.id.0),
752                age_mins = timing.age.num_minutes(),
753                threshold_mins = timing.threshold / 60,
754                "recovering stuck task: no session found — reclaiming early → new"
755            );
756        }
757        // For stuck tasks with an active session, record agent/model failure + cooldown so the
758        // router picks a different agent/model on the next attempt. Without this, the router
759        // sees the agent as healthy and selects it again, causing the hang to repeat.
760        // (Mirrors the pattern in tick_detect_silent_agents lines 219–284.)
761        let mut cached_store_id: Option<i64> = None;
762        if timing.has_session {
763            let store_task = match store.resolve_task_id(repo, &task.id.0).await {
764                Ok(Some(store_id)) => {
765                    cached_store_id = Some(store_id);
766                    match store.get(store_id).await {
767                        Ok(t) => Some(t),
768                        Err(e) => {
769                            tracing::warn!(task_id = task.id.0, error = %e, "failed to fetch task from store for stuck-task cooldown");
770                            None
771                        }
772                    }
773                }
774                Ok(None) => None,
775                Err(e) => {
776                    tracing::warn!(task_id = task.id.0, error = %e, "failed to resolve task id for stuck-task cooldown");
777                    None
778                }
779            };
780            let agent_name = store_task
781                .as_ref()
782                .and_then(|t| t.agent.clone())
783                .unwrap_or_default();
784            let model_name = store_task
785                .as_ref()
786                .and_then(|t| t.model.clone())
787                .unwrap_or_default();
788
789            if !agent_name.is_empty() && !model_name.is_empty() {
790                set_model_cooldown(
791                    &agent_name,
792                    &model_name,
793                    config.silence_cooldown,
794                    "silence_detected",
795                )
796                .await;
797                record_agent_failure_with_message(
798                    &agent_name,
799                    &format!(
800                        "stuck with active session after {}m",
801                        timing.age.num_minutes()
802                    ),
803                )
804                .await;
805            }
806            if !agent_name.is_empty() {
807                set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
808            }
809            tracing::warn!(
810                task_id = task.id.0,
811                agent = %agent_name,
812                model = %model_name,
813                cooldown_secs = config.silence_cooldown,
814                "stuck-task cooldown applied — router will avoid this agent/model on next attempt"
815            );
816        }
817
818        // Remove stale agent/model labels so the LLM router re-routes properly.
819        // Fire-and-forget: cosmetic label operation, tick should not block on it.
820        {
821            let stale_labels: Vec<String> = task
822                .labels
823                .iter()
824                .filter(|l| l.starts_with("agent:") || l.starts_with("model:"))
825                .cloned()
826                .collect();
827            if !stale_labels.is_empty() {
828                let backend_clone = Arc::clone(backend);
829                let task_id_clone = task.id.clone();
830                tokio::spawn(async move {
831                    for label in &stale_labels {
832                        if let Err(e) = backend_clone.remove_label(&task_id_clone, label).await {
833                            tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during stuck-task recovery");
834                        }
835                    }
836                });
837            }
838        }
839        let resolved_store_id = match cached_store_id {
840            Some(id) => Some(id),
841            None => match store.resolve_task_id(repo, &task.id.0).await {
842                Ok(Some(id)) => Some(id),
843                Ok(None) => {
844                    tracing::warn!(
845                        task_id = %task.id.0,
846                        repo,
847                        "resolve_task_id returned None during stuck-task recovery"
848                    );
849                    None
850                }
851                Err(e) => {
852                    tracing::error!(
853                        task_id = %task.id.0,
854                        repo,
855                        error = %e,
856                        "resolve_task_id failed during stuck-task recovery"
857                    );
858                    None
859                }
860            },
861        };
862        if let Some(store_id) = resolved_store_id {
863            if let Err(e) = store_set_by_id(
864                Some(store),
865                store_id,
866                &[
867                    ("agent", serde_json::Value::Null),
868                    ("model", serde_json::Value::Null),
869                    ("route_attempts", serde_json::json!(0)),
870                ],
871            )
872            .await
873            {
874                tracing::warn!(
875                    task_id = task.id.0,
876                    ?e,
877                    "failed to clear routing fields for stuck task — will retry next tick"
878                );
879                continue;
880            }
881            finalize_open_runs_for_recovery(
882                store,
883                store_id,
884                if timing.has_session {
885                    "stuck-task recovery: active session killed"
886                } else {
887                    "stuck-task recovery: no session found"
888                },
889            )
890            .await;
891        }
892        if let Err(e) = task_manager.update_task_status(&task.id, Status::New).await {
893            tracing::warn!(task_id = task.id.0, ?e, "failed to reset stuck task status");
894            continue;
895        }
896
897        // Log reroute activity for stuck task recovery
898        let details = serde_json::json!({
899            "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
900            "age_minutes": timing.age.num_minutes(),
901            "had_session": timing.has_session,
902        });
903        crate::store::store_log_activity(
904            &Some(Arc::clone(store)),
905            repo,
906            &task.id.0,
907            "rerouted",
908            Some("in_progress"),
909            Some("new"),
910            None::<&str>,
911            None::<&str>,
912            Some(&details),
913        )
914        .await;
915
916        let reason = if timing.has_session {
917            format!(
918                "timed out after {}m with active session (cleared agent for re-routing)",
919                timing.age.num_minutes()
920            )
921        } else {
922            format!(
923                "no session found — reclaiming early after {}m (cleared agent for re-routing)",
924                timing.age.num_minutes()
925            )
926        };
927        // Fire-and-forget: comment is informational — tick should not block on it.
928        {
929            let backend_clone = Arc::clone(backend);
930            let task_id_clone = task.id.clone();
931            let body = format!(
932                "[{}] recovered: stuck in_progress — {}{}",
933                chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
934                reason,
935                crate::engine::orch_footer()
936            );
937            tokio::spawn(async move {
938                if let Err(e) = backend_clone.post_comment(&task_id_clone, &body).await {
939                    tracing::warn!(
940                        task_id = task_id_clone.0,
941                        ?e,
942                        "failed to post stuck-task recovery comment"
943                    );
944                }
945            });
946        }
947    }
948
949    let mut routed = match task_manager.list_external_by_status(Status::Routed).await {
950        Ok(tasks) => tasks,
951        Err(e) => {
952            tracing::warn!(?e, "failed to list routed tasks for stuck recovery");
953            vec![]
954        }
955    };
956    match task_manager
957        .list_internal_by_status(crate::store::TaskStatus::Routed)
958        .await
959    {
960        Ok(tasks) => routed.extend(tasks),
961        Err(e) => tracing::warn!(
962            ?e,
963            "failed to list internal routed tasks for stuck recovery"
964        ),
965    }
966    for task in &routed {
967        let Some(timing) = stuck_task_timing_from_map(
968            tmux,
969            repo,
970            &task.id.0,
971            &task.updated_at,
972            config,
973            "cannot parse updated_at, skipping routed stuck-task check",
974            session_map,
975        ) else {
976            continue;
977        };
978
979        if !timing.has_session {
980            continue;
981        }
982
983        let session_name = tmux.session_name(repo, &task.id.0);
984        if let Err(e) = tmux.kill_session(&session_name).await {
985            tracing::warn!(
986                task_id = task.id.0,
987                session = %session_name,
988                error = %e,
989                "failed to kill stale tmux session blocking routed task"
990            );
991            continue;
992        }
993
994        recover_routed_blocked_dispatch(backend, store, task_manager, repo, task, &timing).await;
995    }
996
997    // Recover internal (SQLite) tasks stuck in in_progress.
998    // These have no GitHub labels or comments — just reset the DB status to New.
999    let internal_in_progress = match task_manager
1000        .list_internal_by_status(DbStatus::InProgress)
1001        .await
1002    {
1003        Ok(tasks) => tasks,
1004        Err(e) => {
1005            tracing::warn!(
1006                ?e,
1007                "failed to list internal in_progress tasks for stuck recovery"
1008            );
1009            vec![]
1010        }
1011    };
1012    for task in &internal_in_progress {
1013        let task_id = task.id.0.clone();
1014        let Some(timing) = stuck_task_timing_from_map(
1015            tmux,
1016            repo,
1017            &task_id,
1018            &task.updated_at,
1019            config,
1020            "cannot parse updated_at, skipping stuck internal-task check",
1021            session_map,
1022        ) else {
1023            continue;
1024        };
1025
1026        // See the comment on the external in-progress loop above: a dead session with a
1027        // stale `updated_at` also matches the tail end of a run that just finished
1028        // successfully. Skip the reclaim while the runner's dispatch guard for this task
1029        // is still held — this is the exact race that discards completed daily/scheduled
1030        // internal tasks whose total runtime exceeds `no_session_stuck_timeout`.
1031        if !timing.has_session {
1032            let dispatch_key = format!("{repo}/{task_id}");
1033            if dispatching.contains_key(&dispatch_key) {
1034                // Promoted from debug! to info! (issue #3526): the default log filter is
1035                // "orch=info" (main.rs), so this line — along with the DispatchGuard
1036                // claim/release lines in dispatch_guard.rs and dispatch_tasks_for_repo —
1037                // never reached orch.log in production. Every occurrence of the reclaim
1038                // race (#3518, #3519, #3523, #3525, #3526) had to be reasoned about from
1039                // timestamp correlation across unrelated log lines instead of directly
1040                // observing whether the guard was held, because the one log line that
1041                // would prove it was silently filtered out. Promoting these closes that
1042                // observability gap so the next occurrence carries conclusive evidence.
1043                tracing::info!(
1044                    task_id,
1045                    dispatch_key,
1046                    age_mins = timing.age.num_minutes(),
1047                    "skipping stuck-task reclaim: runner still actively processing this task"
1048                );
1049                continue;
1050            }
1051        }
1052
1053        if timing.has_session {
1054            tracing::warn!(
1055                task_id,
1056                age_mins = timing.age.num_minutes(),
1057                threshold_mins = timing.threshold / 60,
1058                "recovering stuck task: timed out with active session → new"
1059            );
1060        } else {
1061            tracing::warn!(
1062                task_id,
1063                dispatch_key = format!("{repo}/{task_id}"),
1064                age_mins = timing.age.num_minutes(),
1065                threshold_mins = timing.threshold / 60,
1066                "recovering stuck task: no session found — reclaiming early → new"
1067            );
1068        }
1069        // Apply agent/model cooldown for internal stuck tasks with active sessions,
1070        // mirroring the external path (lines 501-546). Without this, the router picks
1071        // the same agent/model that caused the hang, creating an infinite loop.
1072        let mut cached_store_id: Option<i64> = None;
1073        if timing.has_session {
1074            let store_task = match store.resolve_task_id(repo, &task_id).await {
1075                Ok(Some(store_id)) => {
1076                    cached_store_id = Some(store_id);
1077                    match store.get(store_id).await {
1078                        Ok(t) => Some(t),
1079                        Err(e) => {
1080                            tracing::warn!(task_id, error = %e, "failed to fetch task from store for stuck internal-task cooldown");
1081                            None
1082                        }
1083                    }
1084                }
1085                Ok(None) => None,
1086                Err(e) => {
1087                    tracing::warn!(task_id, error = %e, "failed to resolve task id for stuck internal-task cooldown");
1088                    None
1089                }
1090            };
1091            let agent_name = store_task
1092                .as_ref()
1093                .and_then(|t| t.agent.clone())
1094                .unwrap_or_default();
1095            let model_name = store_task
1096                .as_ref()
1097                .and_then(|t| t.model.clone())
1098                .unwrap_or_default();
1099
1100            if !agent_name.is_empty() && !model_name.is_empty() {
1101                set_model_cooldown(
1102                    &agent_name,
1103                    &model_name,
1104                    config.silence_cooldown,
1105                    "silence_detected",
1106                )
1107                .await;
1108                record_agent_failure_with_message(
1109                    &agent_name,
1110                    &format!(
1111                        "internal task stuck with active session after {}m",
1112                        timing.age.num_minutes()
1113                    ),
1114                )
1115                .await;
1116            }
1117            if !agent_name.is_empty() {
1118                set_agent_cooldown(&agent_name, SILENCE_AGENT_COOLDOWN_SECS).await;
1119            }
1120            tracing::warn!(
1121                task_id,
1122                agent = %agent_name,
1123                model = %model_name,
1124                cooldown_secs = config.silence_cooldown,
1125                "internal stuck-task cooldown applied — router will avoid this agent/model on next attempt"
1126            );
1127        }
1128        // Reset routing state so the LLM router is used on the next attempt
1129        // (same reset that external tasks perform).
1130        let resolved_store_id = match cached_store_id {
1131            Some(id) => Some(id),
1132            None => match store.resolve_task_id(repo, &task_id).await {
1133                Ok(Some(id)) => Some(id),
1134                Ok(None) => {
1135                    tracing::warn!(
1136                        task_id = %task_id,
1137                        repo,
1138                        "resolve_task_id returned None during stuck internal-task recovery"
1139                    );
1140                    None
1141                }
1142                Err(e) => {
1143                    tracing::error!(
1144                        task_id = %task_id,
1145                        repo,
1146                        error = %e,
1147                        "resolve_task_id failed during stuck internal-task recovery"
1148                    );
1149                    None
1150                }
1151            },
1152        };
1153        if let Some(store_id) = resolved_store_id {
1154            if let Err(e) = store_set_by_id(
1155                Some(store),
1156                store_id,
1157                &[
1158                    ("agent", serde_json::Value::Null),
1159                    ("model", serde_json::Value::Null),
1160                    ("route_attempts", serde_json::json!(0)),
1161                ],
1162            )
1163            .await
1164            {
1165                tracing::warn!(
1166                    task_id,
1167                    ?e,
1168                    "failed to clear routing fields for stuck internal task — will retry next tick"
1169                );
1170                continue;
1171            }
1172            finalize_open_runs_for_recovery(
1173                store,
1174                store_id,
1175                if timing.has_session {
1176                    "stuck-task recovery: internal active session killed"
1177                } else {
1178                    "stuck-task recovery: internal no session found"
1179                },
1180            )
1181            .await;
1182        }
1183        if let Err(e) = task_manager
1184            .update_task_status(&ExternalId(task_id.clone()), Status::New)
1185            .await
1186        {
1187            tracing::warn!(task_id, ?e, "failed to reset stuck internal task status");
1188        }
1189    }
1190
1191    // Recover tasks stuck in in_review, external and internal in one sweep. Reset to NeedsReview
1192    // (not New) so the review agent re-triggers without clearing the routing state.
1193    //
1194    // This is the only periodic owner of the case where the review session is gone. The sync
1195    // sweep used to reset the same tasks after one minute, which beat a finished review's result
1196    // to the engine and dropped it as stale, and left this longer limit unreachable
1197    let in_review = match store
1198        .list_by_status(repo, crate::store::TaskStatus::InReview)
1199        .await
1200    {
1201        Ok(v) => v,
1202        Err(e) => {
1203            tracing::warn!(?e, "failed to list in_review tasks for stuck recovery");
1204            vec![]
1205        }
1206    };
1207    // A review agent ends its tmux session before its result reaches the engine, so the
1208    // no-session limit for a review has to be longer than for a task agent. The floor in
1209    // `EngineConfig::from_config` keeps it above the review agent's own timeout
1210    let in_review_config = crate::engine::EngineConfig {
1211        no_session_stuck_timeout: config.in_review_no_session_stuck_timeout,
1212        ..config.clone()
1213    };
1214    for task in &in_review {
1215        let Some(task_id) = task.external_id.clone() else {
1216            tracing::warn!(
1217                store_id = task.id,
1218                "in_review task has no external id, skipping stuck recovery"
1219            );
1220            continue;
1221        };
1222
1223        // The dispatch flow owns a task while it holds the guard
1224        if dispatching.contains_key(&format!("{repo}/{task_id}")) {
1225            tracing::debug!(
1226                task_id,
1227                "task locked by dispatch flow, skipping stuck check"
1228            );
1229            continue;
1230        }
1231
1232        // A recorded decision means the review finished and the task waits on the merge, not on
1233        // a review agent
1234        let (entered_at, decided) = in_review_stuck_entry(store, task.id, &task.updated_at).await;
1235        if decided {
1236            tracing::debug!(
1237                task_id,
1238                "in_review task already has a review decision, waiting on merge"
1239            );
1240            continue;
1241        }
1242
1243        let review_task_id = format!("{task_id}-review");
1244        let Some(timing) = stuck_task_timing_from_map(
1245            tmux,
1246            repo,
1247            &review_task_id,
1248            &entered_at,
1249            &in_review_config,
1250            "cannot parse in_review timestamp, skipping stuck in_review check",
1251            session_map,
1252        ) else {
1253            continue;
1254        };
1255
1256        tracing::warn!(
1257            task_id,
1258            age_mins = timing.age.num_minutes(),
1259            threshold_mins = timing.threshold / 60,
1260            has_session = timing.has_session,
1261            "recovering stuck in_review task — resetting to needs_review"
1262        );
1263
1264        match task_manager
1265            .update_task_status_if(
1266                &ExternalId(task_id.clone()),
1267                Status::NeedsReview,
1268                Status::InReview,
1269            )
1270            .await
1271        {
1272            Err(e) => {
1273                tracing::warn!(task_id, ?e, "failed to reset stuck in_review task status");
1274                continue;
1275            }
1276            Ok(false) => {
1277                tracing::debug!(
1278                    task_id,
1279                    "stuck in_review reset skipped — task already transitioned"
1280                );
1281                continue;
1282            }
1283            Ok(true) => {}
1284        }
1285
1286        // A dead or hung session is an infrastructure event, a tmux crash or a restart, not an
1287        // agent parse failure. Keeping the counter would spend the next cycle's quota
1288        if let Err(e) = store
1289            .set_fields(task.id, &[("review_agent_failures", serde_json::json!(0))])
1290            .await
1291        {
1292            tracing::warn!(task_id, ?e, "failed to reset review_agent_failures");
1293        }
1294
1295        finalize_open_runs_for_recovery(
1296            store,
1297            task.id,
1298            if timing.has_session {
1299                "stuck-task recovery: in_review session killed"
1300            } else {
1301                "stuck-task recovery: in_review no session found"
1302            },
1303        )
1304        .await;
1305
1306        let details = serde_json::json!({
1307            "failure_reason": "stuck_in_review",
1308            "age_minutes": timing.age.num_minutes(),
1309            "had_session": timing.has_session,
1310        });
1311        crate::store::store_log_activity(
1312            &Some(Arc::clone(store)),
1313            repo,
1314            &task_id,
1315            "rerouted",
1316            Some("in_review"),
1317            Some("needs_review"),
1318            None::<&str>,
1319            None::<&str>,
1320            Some(&details),
1321        )
1322        .await;
1323
1324        // An internal task has no issue to comment on
1325        if task.origin == "internal" {
1326            continue;
1327        }
1328
1329        if let Err(e) = backend
1330            .post_comment(
1331                &ExternalId(task_id.clone()),
1332                &format!(
1333                    "[{}] recovered: stuck in_review — {} after {}m, resetting to needs_review{}",
1334                    chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
1335                    if timing.has_session {
1336                        "review session hung"
1337                    } else {
1338                        "no session found"
1339                    },
1340                    timing.age.num_minutes(),
1341                    crate::engine::orch_footer()
1342                ),
1343            )
1344            .await
1345        {
1346            tracing::warn!(
1347                task_id,
1348                ?e,
1349                "failed to post stuck in_review recovery comment"
1350            );
1351        }
1352    }
1353
1354    Ok(())
1355}
1356
1357/// External tasks stuck in_progress in a repo no project ticks, such as a project removed from
1358/// the config. Store-wide, so `tick_all` runs it once per tick rather than once per project
1359pub(crate) async fn recover_untracked_repo_stuck_tasks(
1360    tmux: &Arc<TmuxManager>,
1361    config: &EngineConfig,
1362    store: &Arc<TaskStore>,
1363    session_map: &std::collections::HashMap<String, bool>,
1364    dispatching: &Arc<DashMap<String, String>>,
1365    active_repos: &std::collections::HashSet<String>,
1366) {
1367    let all_external_in_progress = match store
1368        .list_all_by_status_global(crate::store::TaskStatus::InProgress)
1369        .await
1370    {
1371        Ok(tasks) => tasks,
1372        Err(e) => {
1373            tracing::warn!(
1374                ?e,
1375                "failed to list global in_progress tasks for cross-repo stuck recovery"
1376            );
1377            vec![]
1378        }
1379    };
1380    for task in all_external_in_progress
1381        .iter()
1382        .filter(|t| t.origin != "internal" && !active_repos.contains(&t.repo))
1383    {
1384        let external_id = task
1385            .external_id
1386            .clone()
1387            .unwrap_or_else(|| task.id.to_string());
1388        let Some(timing) = stuck_task_timing_from_map(
1389            tmux,
1390            &task.repo,
1391            &external_id,
1392            &task.updated_at,
1393            config,
1394            "cannot parse updated_at, skipping cross-repo stuck-task check",
1395            session_map,
1396        ) else {
1397            continue;
1398        };
1399
1400        // See the comment on the in-progress loop in `tick_recover_stuck_tasks`: a dead session with a stale
1401        // `updated_at` also matches the tail end of a run that just finished successfully.
1402        // Skip the reclaim while the runner's dispatch guard for this task is still held.
1403        if !timing.has_session {
1404            let dispatch_key = format!("{}/{external_id}", task.repo);
1405            if dispatching.contains_key(&dispatch_key) {
1406                // See the same promotion note on the internal-task loop in `tick_recover_stuck_tasks`: this was
1407                // debug! (silently dropped under the default "orch=info" filter), leaving
1408                // every prior reclaim-race occurrence undiagnosable from logs alone.
1409                tracing::info!(
1410                    task_id = task.id,
1411                    repo = %task.repo,
1412                    dispatch_key,
1413                    age_mins = timing.age.num_minutes(),
1414                    "skipping cross-repo stuck-task reclaim: runner still actively processing this task"
1415                );
1416                continue;
1417            }
1418        }
1419
1420        if timing.has_session {
1421            tracing::warn!(
1422                task_id = task.id,
1423                repo = %task.repo,
1424                age_mins = timing.age.num_minutes(),
1425                threshold_mins = timing.threshold / 60,
1426                "recovering cross-repo stuck task: timed out with active session → new"
1427            );
1428        } else {
1429            tracing::warn!(
1430                task_id = task.id,
1431                repo = %task.repo,
1432                dispatch_key = format!("{}/{external_id}", task.repo),
1433                age_mins = timing.age.num_minutes(),
1434                threshold_mins = timing.threshold / 60,
1435                "recovering cross-repo stuck task: no session found — reclaiming early → new"
1436            );
1437        }
1438
1439        if let Err(e) = store_set_by_id(
1440            Some(store),
1441            task.id,
1442            &[
1443                ("agent", serde_json::Value::Null),
1444                ("model", serde_json::Value::Null),
1445                ("route_attempts", serde_json::json!(0)),
1446            ],
1447        )
1448        .await
1449        {
1450            tracing::warn!(
1451                task_id = task.id,
1452                repo = %task.repo,
1453                ?e,
1454                "failed to clear routing fields for cross-repo stuck task — will retry next tick"
1455            );
1456            continue;
1457        }
1458
1459        finalize_open_runs_for_recovery(
1460            store,
1461            task.id,
1462            if timing.has_session {
1463                "stuck-task recovery: cross-repo active session killed"
1464            } else {
1465                "stuck-task recovery: cross-repo no session found"
1466            },
1467        )
1468        .await;
1469
1470        if let Err(e) = store
1471            .update_status(task.id, crate::store::TaskStatus::New)
1472            .await
1473        {
1474            tracing::warn!(
1475                task_id = task.id,
1476                repo = %task.repo,
1477                ?e,
1478                "failed to reset cross-repo stuck task status"
1479            );
1480            continue;
1481        }
1482
1483        let details = serde_json::json!({
1484            "failure_reason": if timing.has_session { "stuck_with_session" } else { "stuck_no_session" },
1485            "age_minutes": timing.age.num_minutes(),
1486            "had_session": timing.has_session,
1487            "scope": "cross_repo_global_sweep",
1488        });
1489        crate::store::store_log_activity(
1490            &Some(Arc::clone(store)),
1491            &task.repo,
1492            &external_id,
1493            "rerouted",
1494            Some("in_progress"),
1495            Some("new"),
1496            None::<&str>,
1497            None::<&str>,
1498            Some(&details),
1499        )
1500        .await;
1501    }
1502}
1503
1504/**
1505 * The three store-wide recovery sweeps, run once per tick rather than once per project.
1506 *
1507 * Each one reads every repo, so a per-project call duplicates the work. The NeedsReview
1508 * sweep also reads `needs_review_refires` and then increments it, so concurrent per-project
1509 * passes all see the same count and four projects burn four of the five refire attempts in
1510 * one tick (#3864)
1511 **/
1512pub(crate) async fn tick_global_sweeps(
1513    task_manager: &Arc<TaskManager>,
1514    store: &Arc<TaskStore>,
1515    router: &Router,
1516    dispatching: &Arc<DashMap<String, String>>,
1517    auto_merge_in_flight: &Arc<DashSet<String>>,
1518) {
1519    // Global sweep for CI-failure blocked tasks from inactive or removed projects.
1520    // auto_unblock_blocked_tasks runs inside sync_tick and is scoped to the active repo.
1521    // Tasks blocked for projects no longer in orch config are never processed there.
1522    // This sweep covers all repos so stale CI-failure blocks are eventually resolved.
1523    if let Err(e) = crate::engine::sync::auto_unblock_ci_failure_blocked_tasks_global(
1524        task_manager,
1525        store,
1526        auto_merge_in_flight,
1527    )
1528    .await
1529    {
1530        tracing::warn!(err = %e, "global CI-failure blocked sweep failed");
1531    }
1532
1533    // Re-fire stale NeedsReview tasks and escalate persistently-stuck ones to
1534    // Blocked, across all repos
1535    if let Err(e) = crate::engine::sync::refire_and_escalate_stale_needs_review_global(
1536        task_manager,
1537        store,
1538        router,
1539        dispatching,
1540    )
1541    .await
1542    {
1543        tracing::warn!(err = %e, "global NeedsReview refire/escalation sweep failed");
1544    }
1545
1546    // Recover rebroadcast-blocked tasks once a review agent is routable, across
1547    // all repos
1548    if let Err(e) = crate::engine::sync::auto_recover_rebroadcast_blocked_tasks_global(
1549        router,
1550        task_manager,
1551        store,
1552    )
1553    .await
1554    {
1555        tracing::warn!(err = %e, "global rebroadcast-blocked recovery sweep failed");
1556    }
1557}
1558
1559/// Phase 3a of tick: route status:new tasks to an agent and transition them to status:routed.
1560pub(crate) async fn tick_route_tasks(
1561    backend: &Arc<dyn ExternalBackend>,
1562    task_manager: &Arc<TaskManager>,
1563    router: &mut Router,
1564    store: &Arc<TaskStore>,
1565    repo: &str,
1566    routing_quota: &mut RoutingTickQuota,
1567) -> anyhow::Result<()> {
1568    let _span = tracing::info_span!("engine.tick.phase3a.route").entered();
1569    if let Err(err) = crate::engine::dependencies::apply_claim_gates(store).await {
1570        tracing::warn!(err = %err, "dependency claim gate failed");
1571    }
1572    if routing_quota.is_exhausted() {
1573        tracing::debug!("routing quota exhausted for this tick — skipping repo routing phase");
1574        return Ok(());
1575    }
1576
1577    // Global GitHub 5xx circuit breaker — skip routing during sustained GitHub outages
1578    // to avoid routing-heavy retry storms. Tasks will remain in 'new' status and be
1579    // retried when the circuit closes.
1580    if is_github_circuit_open() {
1581        let remaining = github_circuit_remaining_secs();
1582        tracing::info!(
1583            remaining_secs = remaining,
1584            "GitHub 5xx circuit breaker open — skipping routing phase"
1585        );
1586        return Ok(());
1587    }
1588    let new_tasks = task_manager.list_routable().await?;
1589    let routable: Vec<&ExternalTask> = new_tasks
1590        .iter()
1591        .filter(|t| !skips_agent(&t.labels))
1592        .collect();
1593
1594    // Pre-emptive health check: refresh degraded-agent flags once per tick,
1595    // not per-task (previously this ran inside route() causing N redundant
1596    // DB queries when routing N tasks in the same tick).
1597    if !routable.is_empty() {
1598        let start = std::time::Instant::now();
1599        router.refresh_health(store).await;
1600        tracing::debug!(
1601            duration_ms = start.elapsed().as_millis(),
1602            "router.refresh_health completed"
1603        );
1604    }
1605
1606    // Limit routing to the remaining global quota for this outer engine tick.
1607    for task in routable.into_iter().take(routing_quota.remaining()) {
1608        if !routing_quota.consume_one() {
1609            break;
1610        }
1611        let _task_span = tracing::info_span!("engine.route", task_id = %task.id.0).entered();
1612
1613        let task_start = Instant::now();
1614        match router.route(task, store, repo).await {
1615            Ok(result) => {
1616                tracing::debug!(task_id = %task.id.0, duration_ms = task_start.elapsed().as_millis(), "route completed");
1617                // Store route result in store
1618                if let Err(e) = router
1619                    .store_route_result(&task.id.0, &result, store, repo)
1620                    .await
1621                {
1622                    tracing::warn!(
1623                        task_id = task.id.0,
1624                        ?e,
1625                        "failed to store route result; skipping Routed transition"
1626                    );
1627                    continue;
1628                }
1629
1630                if is_internal_id(&task.id.0) {
1631                    // Internal tasks: update DB status, skip GitHub label ops.
1632                    if let Err(e) = task_manager
1633                        .update_task_status(&task.id, Status::Routed)
1634                        .await
1635                    {
1636                        tracing::warn!(
1637                            task_id = task.id.0,
1638                            ?e,
1639                            "failed to set internal status:routed"
1640                        );
1641                        continue;
1642                    }
1643                } else {
1644                    // Fire-and-forget: label operations are cosmetic (routing already
1645                    // persisted in DB). Spawning them prevents blocking the tick loop
1646                    // on 6-9 sequential HTTP calls (~3-9s overhead per routed task).
1647                    let backend_clone = Arc::clone(backend);
1648                    let task_id_clone = task.id.clone();
1649                    let task_labels: Vec<String> = task.labels.clone();
1650                    let new_labels = {
1651                        let mut labels = vec![
1652                            format!("agent:{}", result.agent),
1653                            format!("complexity:{}", result.complexity),
1654                        ];
1655                        if let Some(ref model) = result.model {
1656                            labels.push(format!("model:{model}"));
1657                        }
1658                        labels
1659                    };
1660                    tokio::spawn(async move {
1661                        // Remove old agent/complexity/model labels
1662                        for label in &task_labels {
1663                            if (label.starts_with("agent:")
1664                                || label.starts_with("complexity:")
1665                                || label.starts_with("model:"))
1666                                && let Err(e) =
1667                                    backend_clone.remove_label(&task_id_clone, label).await
1668                            {
1669                                tracing::warn!(task_id = task_id_clone.0, label, error = %e, "failed to remove stale routing label during re-route");
1670                            }
1671                        }
1672                        // Add new labels
1673                        if let Err(e) = backend_clone.set_labels(&task_id_clone, &new_labels).await
1674                        {
1675                            tracing::warn!(
1676                                task_id = task_id_clone.0,
1677                                ?e,
1678                                "failed to set routing labels"
1679                            );
1680                        }
1681                    });
1682
1683                    // Transition to routed
1684                    if let Err(e) = task_manager
1685                        .update_task_status(&task.id, Status::Routed)
1686                        .await
1687                    {
1688                        tracing::warn!(task_id = task.id.0, ?e, "failed to set status:routed");
1689                        continue;
1690                    }
1691                }
1692
1693                // Dual-write: upsert task + store route result in SQLite
1694                match store.ensure_external_task(repo, task).await {
1695                    Ok(store_id) => {
1696                        let profile_json =
1697                            serde_json::to_string(&result.profile).unwrap_or_default();
1698                        let skills_json =
1699                            serde_json::to_string(&result.selected_skills).unwrap_or_default();
1700                        if let Err(e) = store
1701                            .store_route(&crate::store::StoreRoute {
1702                                id: store_id,
1703                                agent: &result.agent,
1704                                model: result.model.as_deref(),
1705                                complexity: &result.complexity,
1706                                estimate: result.estimate,
1707                                reason: &result.reason,
1708                                profile: &profile_json,
1709                                skills: &skills_json,
1710                            })
1711                            .await
1712                        {
1713                            tracing::debug!(
1714                                task_id = task.id.0,
1715                                ?e,
1716                                "dual-write: store_route failed"
1717                            );
1718                        }
1719                        if let Err(e) = store
1720                            .update_status(store_id, crate::store::TaskStatus::Routed)
1721                            .await
1722                        {
1723                            tracing::debug!(
1724                                task_id = task.id.0,
1725                                ?e,
1726                                "dual-write: update_status failed"
1727                            );
1728                        }
1729                        // Sync the estimate to GitHub Projects if the router assigned one.
1730                        // Fire-and-forget: do NOT await inline — GraphQL mutations can take
1731                        // up to 30 s and would block the entire tick loop on every routed task.
1732                        if result.estimate > 0 {
1733                            let backend_clone = Arc::clone(backend);
1734                            let task_id = task.id.clone();
1735                            let estimate = result.estimate;
1736                            tokio::spawn(async move {
1737                                if let Err(e) = backend_clone
1738                                    .sync_estimate_to_project(&task_id, estimate)
1739                                    .await
1740                                {
1741                                    tracing::debug!(
1742                                        task_id = task_id.0,
1743                                        estimate,
1744                                        err = %e,
1745                                        "dual-write: estimate project sync failed"
1746                                    );
1747                                }
1748                            });
1749                        }
1750                    }
1751                    Err(e) => {
1752                        tracing::debug!(
1753                            task_id = task.id.0,
1754                            ?e,
1755                            "dual-write: ensure_external_task failed"
1756                        );
1757                    }
1758                }
1759
1760                if let Some(ref warning) = result.warning {
1761                    tracing::warn!(task_id = task.id.0, warning, "routing sanity warning");
1762                }
1763
1764                tracing::info!(
1765                    task_id = task.id.0,
1766                    agent = %result.agent,
1767                    complexity = %result.complexity,
1768                    reason = %result.reason,
1769                    "task routed"
1770                );
1771            }
1772            Err(e) => {
1773                if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1774                    tracing::error!(
1775                        task_id = %task.id.0,
1776                        scope = %cooled.scope(),
1777                        remaining_secs = ?cooled.remaining_secs(),
1778                        degraded = ?cooled.degraded(),
1779                        "ALL AGENTS COOLED — no agent available to route this task. \
1780                         Task stays in 'new' and will be retried on every tick. \
1781                         Check `orch cooldown list` to see what is blocking dispatch."
1782                    );
1783                    continue;
1784                }
1785                if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1786                    tracing::info!(
1787                        task_id = %task.id.0,
1788                        blockers = %waiting.blockers.join(", "),
1789                        "task is waiting on a dependency"
1790                    );
1791                    continue;
1792                }
1793                tracing::error!(task_id = task.id.0, ?e, "routing failed, skipping task");
1794            }
1795        }
1796    }
1797    Ok(())
1798}
1799
1800/// Global routing sweep: route status:new tasks from inactive or removed repos.
1801///
1802/// `tick_route_tasks` is scoped to the active repo per-tick and never processes
1803/// tasks whose `repo` field differs from the current project. When a project is
1804/// removed from the orch config, its tasks accumulate in `new` indefinitely
1805/// because no tick ever runs for that repo. This sweep covers all repos so
1806/// orphaned new tasks are eventually routed.
1807///
1808/// `active_repos` must be the full set of currently-configured project repos,
1809/// not just the repo of the tick that called this — mirroring
1810/// `dispatch_routed_tasks_global` (see its doc comment for the "racing the real
1811/// per-repo dispatch" failure mode this guards against). Filtering on
1812/// `t.repo != current_repo` alone (the previous behavior) made every *other*
1813/// active repo's new tasks look "orphaned" too and routed them a tick early,
1814/// through the wrong repo's tick.
1815pub(crate) async fn route_new_tasks_global(
1816    router: &mut Router,
1817    store: &Arc<TaskStore>,
1818    active_repos: &std::collections::HashSet<String>,
1819    routing_quota: &mut RoutingTickQuota,
1820) -> anyhow::Result<()> {
1821    use crate::store::{StoreRoute, TaskStatus};
1822
1823    if routing_quota.is_exhausted() {
1824        tracing::debug!("routing quota exhausted for this tick — skipping global routing sweep");
1825        return Ok(());
1826    }
1827
1828    let all_new = store.list_all_by_status_global(TaskStatus::New).await?;
1829    let orphaned: Vec<&crate::store::Task> = all_new
1830        .iter()
1831        .filter(|t| !active_repos.contains(&t.repo))
1832        .filter(|t| !skips_agent(&t.labels))
1833        .collect();
1834
1835    if orphaned.is_empty() {
1836        return Ok(());
1837    }
1838
1839    tracing::info!(
1840        count = orphaned.len(),
1841        "global routing sweep: found new tasks from inactive repos"
1842    );
1843
1844    router.refresh_health(store).await;
1845
1846    for task in orphaned.into_iter().take(routing_quota.remaining()) {
1847        if !routing_quota.consume_one() {
1848            break;
1849        }
1850        let ext_task = crate::engine::tasks::store_task_to_external(task);
1851        let task_start = Instant::now();
1852        match router.route(&ext_task, store, &task.repo).await {
1853            Ok(result) => {
1854                tracing::debug!(
1855                    task_id = %ext_task.id.0,
1856                    repo = %task.repo,
1857                    duration_ms = task_start.elapsed().as_millis(),
1858                    "global sweep: route completed"
1859                );
1860                let profile_json = serde_json::to_string(&result.profile).unwrap_or_default();
1861                let skills_json =
1862                    serde_json::to_string(&result.selected_skills).unwrap_or_default();
1863                if let Err(e) = store
1864                    .store_route(&StoreRoute {
1865                        id: task.id,
1866                        agent: &result.agent,
1867                        model: result.model.as_deref(),
1868                        complexity: &result.complexity,
1869                        estimate: result.estimate,
1870                        reason: &result.reason,
1871                        profile: &profile_json,
1872                        skills: &skills_json,
1873                    })
1874                    .await
1875                {
1876                    tracing::warn!(
1877                        task_id = %ext_task.id.0,
1878                        repo = %task.repo,
1879                        err = %e,
1880                        "global sweep: failed to store route result; skipping status update"
1881                    );
1882                    continue;
1883                }
1884                match store.update_status(task.id, TaskStatus::Routed).await {
1885                    Err(e) => {
1886                        tracing::warn!(
1887                            task_id = %ext_task.id.0,
1888                            repo = %task.repo,
1889                            err = %e,
1890                            "global sweep: failed to set status:routed"
1891                        );
1892                    }
1893                    _ => {
1894                        tracing::info!(
1895                            task_id = %ext_task.id.0,
1896                            repo = %task.repo,
1897                            agent = %result.agent,
1898                            complexity = %result.complexity,
1899                            "global sweep: task routed"
1900                        );
1901                    }
1902                }
1903                if let Some(ref warning) = result.warning {
1904                    tracing::warn!(task_id = %ext_task.id.0, warning, "global sweep: routing sanity warning");
1905                }
1906            }
1907            Err(e) => {
1908                if let Some(cooled) = e.downcast_ref::<AllAgentsCooledError>() {
1909                    tracing::error!(
1910                        task_id = %ext_task.id.0,
1911                        repo = %task.repo,
1912                        scope = %cooled.scope(),
1913                        remaining_secs = ?cooled.remaining_secs(),
1914                        degraded = ?cooled.degraded(),
1915                        "global sweep: ALL AGENTS COOLED — task stays in 'new'"
1916                    );
1917                    continue;
1918                }
1919                if let Some(waiting) = e.downcast_ref::<crate::store::WaitingOnDependency>() {
1920                    tracing::info!(
1921                        task_id = %ext_task.id.0,
1922                        repo = %task.repo,
1923                        blockers = %waiting.blockers.join(", "),
1924                        "global sweep: task is waiting on a dependency"
1925                    );
1926                    continue;
1927                }
1928                tracing::error!(
1929                    task_id = %ext_task.id.0,
1930                    repo = %task.repo,
1931                    err = ?e,
1932                    "global sweep: routing failed, skipping task"
1933                );
1934            }
1935        }
1936    }
1937
1938    Ok(())
1939}
1940
1941/// Phase 3b of tick: spawn agents for all status:routed tasks up to the parallel limit.
1942#[derive(Debug, Clone, Copy)]
1943pub(crate) struct DispatchMode {
1944    pub(crate) is_degraded: bool,
1945    pub(crate) healthy_agents: usize,
1946    pub(crate) threshold: usize,
1947}
1948
1949pub(crate) fn dispatch_mode_from_router(router: &Router) -> DispatchMode {
1950    let threshold = crate::engine::router::config::min_healthy_agents_threshold();
1951    let healthy_agents = router.healthy_agent_count("simple");
1952    let is_degraded = router.is_degraded(threshold);
1953    DispatchMode {
1954        is_degraded,
1955        healthy_agents,
1956        threshold,
1957    }
1958}
1959
1960/// Phase 3b of tick: spawn agents for all status:routed tasks up to the parallel limit.
1961#[expect(
1962    clippy::too_many_arguments,
1963    reason = "each argument is engine state the tick passes down"
1964)]
1965#[tracing::instrument(skip_all, name = "engine.tick.phase3b.dispatch")]
1966pub(crate) async fn tick_dispatch_tasks(
1967    backend: &Arc<dyn ExternalBackend>,
1968    tmux: &Arc<TmuxManager>,
1969    repo: &str,
1970    runner: &Arc<TaskRunner>,
1971    capture: &Arc<CaptureService>,
1972    semaphore: &Arc<Semaphore>,
1973    task_manager: &Arc<TaskManager>,
1974    weight_tx: &mpsc::Sender<WeightSignal>,
1975    dispatch_mode: DispatchMode,
1976    dispatching: &Arc<DashMap<String, String>>,
1977    store: &Arc<TaskStore>,
1978    session_map: &std::collections::HashMap<String, bool>,
1979) -> anyhow::Result<()> {
1980    // Phase 3a already drops no-agent and owner:human, this filter stays as defense in depth
1981    let mut routed_tasks = task_manager.list_external_by_status(Status::Routed).await?;
1982
1983    // Also include internal tasks in Routed status.
1984    use crate::store::TaskStatus as DbStatus;
1985    let internal_routed = task_manager
1986        .list_internal_by_status(DbStatus::Routed)
1987        .await?;
1988    routed_tasks.extend(internal_routed);
1989
1990    // Filter and collect owned tasks to avoid lifetime issues
1991    let dispatchable: Vec<ExternalTask> = routed_tasks
1992        .into_iter()
1993        .filter(|t| !skips_agent(&t.labels))
1994        .collect();
1995
1996    dispatch_tasks_for_repo(
1997        backend,
1998        tmux,
1999        repo,
2000        runner,
2001        capture,
2002        semaphore,
2003        task_manager,
2004        weight_tx,
2005        dispatch_mode,
2006        dispatching,
2007        store,
2008        session_map,
2009        dispatchable,
2010    )
2011    .await
2012}
2013
2014/// Dispatch a pre-fetched list of routed tasks for a single repo.
2015///
2016/// Shared by `tick_dispatch_tasks` (active-repo, per-tick dispatch) and
2017/// `dispatch_routed_tasks_global` (inactive-repo sweep) so both paths spawn
2018/// agents through the exact same worktree/tmux/runner logic.
2019#[expect(
2020    clippy::too_many_arguments,
2021    reason = "each argument is engine state the tick passes down"
2022)]
2023async fn dispatch_tasks_for_repo(
2024    backend: &Arc<dyn ExternalBackend>,
2025    tmux: &Arc<TmuxManager>,
2026    repo: &str,
2027    runner: &Arc<TaskRunner>,
2028    capture: &Arc<CaptureService>,
2029    semaphore: &Arc<Semaphore>,
2030    task_manager: &Arc<TaskManager>,
2031    weight_tx: &mpsc::Sender<WeightSignal>,
2032    dispatch_mode: DispatchMode,
2033    dispatching: &Arc<DashMap<String, String>>,
2034    store: &Arc<TaskStore>,
2035    session_map: &std::collections::HashMap<String, bool>,
2036    dispatchable: Vec<ExternalTask>,
2037) -> anyhow::Result<()> {
2038    if dispatchable.is_empty() {
2039        tracing::debug!(count = 0, "dispatchable tasks found");
2040        return Ok(());
2041    }
2042
2043    tracing::info!(count = dispatchable.len(), "dispatchable tasks found");
2044    if dispatch_mode.is_degraded {
2045        tracing::warn!(
2046            healthy_agents = dispatch_mode.healthy_agents,
2047            threshold = dispatch_mode.threshold,
2048            "degraded mode: using sequential dispatch"
2049        );
2050    }
2051
2052    let sequential_delay = if dispatch_mode.is_degraded {
2053        crate::engine::router::config::sequential_dispatch_delay_ms()
2054    } else {
2055        0
2056    };
2057
2058    let mut ranked: Vec<(crate::engine::priority::Priority, ExternalTask)> =
2059        Vec::with_capacity(dispatchable.len());
2060    for task in dispatchable {
2061        let has_pr = match store.resolve_id_by_external(repo, &task.id.0).await {
2062            Ok(Some(id)) => store.get(id).await.is_ok_and(|t| t.pr_number.is_some()),
2063            _ => false,
2064        };
2065        ranked.push((
2066            crate::engine::priority::Priority::from_labels(&task.labels).with_pr(has_pr),
2067            task,
2068        ));
2069    }
2070    crate::engine::priority::sort_by_priority(&mut ranked, |t| t.0);
2071
2072    for (idx, (priority, task)) in ranked.into_iter().enumerate() {
2073        if crate::engine::dependencies::dispatch_is_waiting(store, repo, &task.id.0).await {
2074            continue;
2075        }
2076        // In degraded mode, add delay between dispatches to pace the system
2077        if idx > 0 && dispatch_mode.is_degraded {
2078            let delay_ms = sequential_delay;
2079            tracing::debug!(
2080                task_id = task.id.0,
2081                delay_ms = delay_ms,
2082                "sequential dispatch: waiting before dispatch"
2083            );
2084            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
2085        }
2086        // In-memory guard: prevents double-dispatch due to GitHub API eventual consistency.
2087        // After update_status(InProgress), the label removal fires a webhook that can
2088        // trigger an immediate tick. GitHub's search index may not yet reflect the label
2089        // change, so list_by_status(Routed) can still return this task. The tmux session
2090        // does not exist until the runner completes worktree setup (~10s later), so the
2091        // session_exists check alone is insufficient.
2092        let dispatch_key = format!("{}/{}", repo, task.id.0);
2093        // Atomically claim the dispatch slot.  DashMap lets us distinguish:
2094        //   - same task already in-flight (expected): log debug and skip.
2095        //   - different task holds the same key (should never happen): log warning and skip.
2096        // A plain DashSet::insert would return false for both cases without distinction.
2097        {
2098            use dashmap::mapref::entry::Entry;
2099            match dispatching.entry(dispatch_key.clone()) {
2100                Entry::Occupied(existing) => {
2101                    let existing_id = existing.get().clone();
2102                    drop(existing); // release shard lock before logging
2103                    if existing_id == task.id.0 {
2104                        tracing::debug!(
2105                            task_id = task.id.0,
2106                            "task already dispatching, skipping duplicate"
2107                        );
2108                    } else {
2109                        tracing::warn!(
2110                            task_id = task.id.0,
2111                            existing_task_id = existing_id,
2112                            dispatch_key,
2113                            "dispatch key collision: unexpected task already holds this key"
2114                        );
2115                    }
2116                    continue;
2117                }
2118                Entry::Vacant(slot) => {
2119                    slot.insert(task.id.0.clone());
2120                    // Promoted from debug! to info! (issue #3526): paired with the
2121                    // "dispatch guard released" line in dispatch_guard.rs, this is the
2122                    // only direct evidence of how long a dispatch guard was actually held
2123                    // for a given key. Both were debug!, silently dropped under the
2124                    // default "orch=info" filter (main.rs) — see the reclaim-skip log
2125                    // promotions above for the full rationale.
2126                    tracing::info!(task_id = task.id.0, dispatch_key, "dispatch guard claimed");
2127                }
2128            }
2129        }
2130        // RAII guard — removes dispatch_key on drop even if the spawned task panics.
2131        let dispatch_guard =
2132            DispatchGuard::new(std::sync::Arc::clone(dispatching), dispatch_key.clone());
2133        let task_id = task.id.0.clone();
2134
2135        // Load routing result from store (stored during Phase 3a). Loaded before the
2136        // tmux/semaphore/in_progress steps so a stale cooldown can bail out cheaply.
2137        let route_result = match get_route_result(store, repo, &task_id).await {
2138            Ok(r) => Some(r),
2139            Err(RouteResultError::NoAgent { .. }) => {
2140                tracing::warn!(
2141                    task_id,
2142                    "routed task missing agent — resetting to new for re-routing (#1604)"
2143                );
2144                if let Err(e2) = task_manager.update_task_status(&task.id, Status::New).await {
2145                    tracing::error!(task_id, error = %e2, "failed to reset task to new");
2146                }
2147                continue; // dispatch_guard drops here, removing the key
2148            }
2149            Err(e) => {
2150                tracing::warn!(task_id, error = %e, "get_route_result failed — dispatching without route info");
2151                None
2152            }
2153        };
2154
2155        // Re-check the assigned agent/model against the cooldown store: routing and
2156        // dispatch are decoupled by ticks, and a failure recorded after this task was
2157        // routed but before it dispatched would otherwise burn a full run on an
2158        // already-cooled agent/model. Leave the task in Routed so the next tick's
2159        // dispatch pass re-evaluates it — no new per-task defer state (see #3243).
2160        if let Some(route) = route_result.as_ref() {
2161            let agent_cooled = is_agent_in_cooldown(&route.agent);
2162            let model_cooled = route
2163                .model
2164                .as_deref()
2165                .is_some_and(|model| is_model_in_cooldown(&route.agent, model));
2166            if agent_cooled || model_cooled {
2167                tracing::warn!(
2168                    task_id,
2169                    agent = route.agent,
2170                    model = route.model.as_deref().unwrap_or(""),
2171                    "routed agent/model entered cooldown after routing — deferring dispatch to next tick"
2172                );
2173                continue; // dispatch_guard drops here, removing the key
2174            }
2175        }
2176
2177        // Check if already running (has active session)
2178        let session_name = tmux.session_name(repo, &task_id);
2179        if tmux
2180            .session_blocks_dispatch_from_map(&session_name, session_map)
2181            .await
2182        {
2183            tracing::warn!(
2184                task_id,
2185                session_name,
2186                "task has existing tmux session, skipping dispatch"
2187            );
2188            continue; // dispatch_guard drops here, removing the key
2189        }
2190
2191        // Try to acquire a slot
2192        let Some(permit) = crate::engine::priority::acquire_dispatch_slot(semaphore, priority)
2193        else {
2194            tracing::debug!("all parallel slots busy, skipping remaining tasks");
2195            break; // dispatch_guard drops here, removing the key
2196        };
2197
2198        // Mark in_progress BEFORE spawning to prevent double dispatch.
2199        let set_in_progress_result = task_manager
2200            .update_task_status(&task.id, Status::InProgress)
2201            .await;
2202        if let Err(e) = set_in_progress_result {
2203            tracing::error!(task_id, ?e, "failed to set in_progress, skipping dispatch");
2204            drop(permit);
2205            continue; // dispatch_guard drops here, removing the key
2206        }
2207        tracing::info!(task_id, "dispatching task");
2208
2209        // Register session for capture
2210        capture
2211            .register_session(repo, &task_id, &session_name)
2212            .await;
2213
2214        // Dispatch task
2215        let runner = std::sync::Arc::clone(runner);
2216        let backend = std::sync::Arc::clone(backend);
2217        let tmux = std::sync::Arc::clone(tmux);
2218        let capture = std::sync::Arc::clone(capture);
2219        let task_id_for_cleanup = task_id.clone();
2220        let task_owned = task;
2221        let weight_tx = weight_tx.clone();
2222        let repo_owned = repo.to_string();
2223        let task_manager_for_spawn = std::sync::Arc::clone(task_manager);
2224        let store_for_spawn = std::sync::Arc::clone(store);
2225
2226        let repo_ctx = repo_owned.clone();
2227        tokio::spawn(REPO_CONTEXT.scope(repo_ctx, async move {
2228            let _dispatch_guard = dispatch_guard; // released on drop (normal or panic)
2229            let dispatch_start = std::time::Instant::now();
2230            match runner
2231                .run_with_context(&task_owned, &backend, &tmux, route_result.as_ref())
2232                .await
2233            {
2234                Ok(signal) => {
2235                    tracing::info!(task_id, "task runner completed");
2236
2237                    let duration = dispatch_start.elapsed().as_secs_f64();
2238
2239                    // Derive a display status from the weight signal.
2240                    // When review agent is enabled, successful tasks go through
2241                    // review before being marked done — UNLESS the response
2242                    // handler already marked the task done (no-op tasks with
2243                    // no PR and no commits should not enter the review cycle).
2244                    let enable_review = config::get("workflow.enable_review_agent")
2245                        .map_or(true, |v| v != "false");
2246                    let display_status = match &signal {
2247                        WeightSignal::Success { .. } if enable_review => {
2248                            // Check if response handler already set done (no-op task).
2249                            // The task_manager.update_task_status() was already called
2250                            // by response_handler, so read the actual status from store.
2251                            let has_pr = match store_for_spawn
2252                                .resolve_task_id(&repo_owned, &task_id)
2253                                .await
2254                            {
2255                                Ok(Some(store_id)) => match store_for_spawn
2256                                    .get(store_id)
2257                                    .await
2258                                {
2259                                    Ok(t) => t.pr_number.is_some(),
2260                                    Err(e) => {
2261                                        tracing::warn!(task_id, err = %e, "store.get() failed — defaulting to needs_review");
2262                                        true
2263                                    }
2264                                },
2265                                Ok(None) => false,
2266                                Err(e) => {
2267                                    tracing::warn!(task_id, err = %e, "failed to check PR status — defaulting to needs_review");
2268                                    true
2269                                }
2270                            };
2271                            if has_pr {
2272                                "needs_review"
2273                            } else {
2274                                "done"
2275                            }
2276                        }
2277                        // Rerouted: silence detection or other non-rate-limit reset already
2278                        // moved the task to "new". Stay in "new" — never convert to
2279                        // needs_review, which would trigger the no-code shortcut and falsely
2280                        // close the task as done without a successful agent run.
2281                        WeightSignal::RateLimited { .. } | WeightSignal::Rerouted => "new",
2282                        WeightSignal::Blocked => "blocked",
2283                        // None means the runner guard skipped the task (e.g. tmux session
2284                        // already exists) or the task completed with an unrecognised status.
2285                        // Route to needs_review only when the review agent is enabled;
2286                        // otherwise mark done so the task is not permanently stuck.
2287                        WeightSignal::None if enable_review => "needs_review",
2288                        WeightSignal::Success { .. } | WeightSignal::None => "done",
2289                    };
2290
2291                    // Send weight signal back to the router
2292                    let _ = weight_tx.send(signal).await;
2293
2294                    // All tasks: if needs_review, trigger the review agent.
2295                    // Status updates go through task_manager so internal tasks
2296                    // hit SQLite while external tasks hit GitHub labels.
2297                    if display_status == "needs_review" {
2298                        // Transition to NeedsReview — emits event so the review subscriber
2299                        // fires with the correct duration before the review agent starts.
2300                        if let Err(e) = task_manager_for_spawn
2301                            .update_task_status_with_duration(
2302                                &ExternalId(task_id.clone()),
2303                                Status::NeedsReview,
2304                                Some(duration),
2305                            )
2306                            .await
2307                        {
2308                            tracing::error!(task_id, err = %e, "update_task_status(NeedsReview) failed — task may be stuck");
2309                        }
2310                        // Review agent dispatch is handled by the event-driven subscriber.
2311                    } else {
2312                        // done, blocked, or new (rate-limited): update status directly.
2313                        let final_status = match display_status {
2314                            "done" => Status::Done,
2315                            "blocked" => Status::Blocked,
2316                            _ => Status::New,
2317                        };
2318                        if let Err(e) = task_manager_for_spawn
2319                            .update_task_status_with_duration(
2320                                &ExternalId(task_id.clone()),
2321                                final_status,
2322                                Some(duration),
2323                            )
2324                            .await
2325                        {
2326                            tracing::warn!(task_id, ?e, "failed to update task status after completion");
2327                        }
2328                    }
2329                }
2330                Err(e) => {
2331                    tracing::error!(task_id, ?e, "task runner failed");
2332                    let duration = dispatch_start.elapsed().as_secs_f64();
2333                    if is_internal_id(&task_id) {
2334                        // Internal tasks have no GitHub issue to comment on.
2335                        if let Err(ue) = task_manager_for_spawn
2336                            .update_task_status_with_duration(
2337                                &ExternalId(task_id.clone()),
2338                                Status::NeedsReview,
2339                                Some(duration),
2340                            )
2341                            .await
2342                        {
2343                            tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2344                        }
2345                    } else {
2346                        // Post comment (best-effort)
2347                        if let Err(comment_err) = backend
2348                            .post_comment(
2349                                &ExternalId(task_id.clone()),
2350                                &format!(
2351                                    "[{}] error: task runner failed: {e}{}",
2352                                    chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"),
2353                                    crate::engine::orch_footer()
2354                                ),
2355                            )
2356                            .await
2357                        {
2358                            tracing::warn!(
2359                                task_id,
2360                                ?comment_err,
2361                                "failed to post error comment to GitHub"
2362                            );
2363                        }
2364                        // Update status immediately to NeedsReview so external tasks
2365                        // don't remain stuck in InProgress until the no-session timeout.
2366                        // The notify subscriber will send the channel notification with duration.
2367                        if let Err(ue) = task_manager_for_spawn
2368                            .update_task_status_with_duration(
2369                                &ExternalId(task_id.clone()),
2370                                Status::NeedsReview,
2371                                Some(duration),
2372                            )
2373                            .await
2374                        {
2375                            tracing::error!(task_id, err = %ue, "update_task_status(NeedsReview) failed — task may be stuck");
2376                        }
2377                    }
2378                }
2379            }
2380
2381            // Unregister session from capture
2382            capture.unregister_session(&repo_owned, &task_id_for_cleanup).await;
2383
2384            // Release the semaphore permit
2385            drop(permit);
2386            // _dispatch_guard drops here, removing the key from the dispatching set.
2387        }));
2388    }
2389    Ok(())
2390}
2391
2392fn skips_agent(labels: &[String]) -> bool {
2393    labels
2394        .iter()
2395        .any(|label| label == "no-agent" || label == "owner:human")
2396}
2397
2398/// Filter routed tasks down to those belonging to a repo outside `active_repos`,
2399/// drop `no-agent` and `owner:human` tasks, and group the rest by repo.
2400///
2401/// Pure/side-effect-free so it can be unit tested without touching real tmux
2402/// sessions or spawning a runner — see `dispatch_routed_tasks_global`, which is
2403/// the only caller and does the actual (side-effectful) dispatch.
2404fn group_orphaned_routed_tasks(
2405    all_routed: Vec<crate::store::Task>,
2406    active_repos: &std::collections::HashSet<String>,
2407) -> std::collections::HashMap<String, Vec<crate::store::Task>> {
2408    let mut by_repo: std::collections::HashMap<String, Vec<crate::store::Task>> =
2409        std::collections::HashMap::new();
2410    for task in all_routed
2411        .into_iter()
2412        .filter(|t| !active_repos.contains(&t.repo))
2413        .filter(|t| !skips_agent(&t.labels))
2414    {
2415        by_repo.entry(task.repo.clone()).or_default().push(task);
2416    }
2417    by_repo
2418}
2419
2420/// Global dispatch sweep: dispatch status:routed tasks from inactive or removed repos.
2421///
2422/// `tick_dispatch_tasks` is scoped to the active repo per-tick (its `task_manager`
2423/// only queries that repo's tasks) and never processes tasks whose `repo` field
2424/// differs from the current project. When a project is removed from the orch
2425/// config, tasks that reach `routed` for it — including ones routed by the
2426/// `route_new_tasks_global` sweep — have no code path left to dispatch them, so
2427/// they stay `routed` forever. This sweep covers all repos so orphaned routed
2428/// tasks are eventually dispatched.
2429///
2430/// `active_repos` must be the full set of currently-configured project repos,
2431/// not just one project's repo; passing only that project's own repo would make
2432/// every other active repo's routed tasks look "orphaned" too and dispatch them
2433/// a second time through this fallback path, racing the real per-repo dispatch.
2434///
2435/// `TaskRunner::resolve_project_dir` already falls back to the deterministic
2436/// `~/.orch/projects/<owner>/<repo>.git` bare-clone convention when a repo is
2437/// not listed in the active config, so dispatch works without a `ProjectEngine`.
2438#[expect(
2439    clippy::too_many_arguments,
2440    reason = "each argument is engine state the tick passes down"
2441)]
2442pub(crate) async fn dispatch_routed_tasks_global(
2443    tmux: &Arc<TmuxManager>,
2444    capture: &Arc<CaptureService>,
2445    semaphore: &Arc<Semaphore>,
2446    weight_tx: &mpsc::Sender<WeightSignal>,
2447    dispatch_mode: DispatchMode,
2448    dispatching: &Arc<DashMap<String, String>>,
2449    store: &Arc<TaskStore>,
2450    session_map: &std::collections::HashMap<String, bool>,
2451    active_repos: &std::collections::HashSet<String>,
2452) -> anyhow::Result<()> {
2453    use crate::store::TaskStatus;
2454
2455    let all_routed = store.list_all_by_status_global(TaskStatus::Routed).await?;
2456    let by_repo = group_orphaned_routed_tasks(all_routed, active_repos);
2457
2458    if by_repo.is_empty() {
2459        return Ok(());
2460    }
2461
2462    tracing::info!(
2463        repos = by_repo.len(),
2464        "global dispatch sweep: found routed tasks from inactive repos"
2465    );
2466
2467    for (repo, tasks) in by_repo {
2468        let backend: Arc<dyn ExternalBackend> = match crate::backends::github::GitHubBackend::new(
2469            repo.clone(),
2470        ) {
2471            Ok(b) => Arc::new(b),
2472            Err(e) => {
2473                tracing::warn!(repo = %repo, err = %e, "global sweep: failed to construct backend, skipping repo");
2474                continue;
2475            }
2476        };
2477        let task_manager = Arc::new(TaskManager::with_store(
2478            std::sync::Arc::clone(&backend),
2479            std::sync::Arc::clone(store),
2480            repo.clone(),
2481        ));
2482        let runner =
2483            Arc::new(TaskRunner::new(repo.clone()).with_store(std::sync::Arc::clone(store)));
2484        let dispatchable: Vec<ExternalTask> = tasks
2485            .iter()
2486            .map(crate::engine::tasks::store_task_to_external)
2487            .collect();
2488
2489        if let Err(e) = dispatch_tasks_for_repo(
2490            &backend,
2491            tmux,
2492            &repo,
2493            &runner,
2494            capture,
2495            semaphore,
2496            &task_manager,
2497            weight_tx,
2498            dispatch_mode,
2499            dispatching,
2500            store,
2501            session_map,
2502            dispatchable,
2503        )
2504        .await
2505        {
2506            tracing::warn!(repo = %repo, err = %e, "global sweep: dispatch failed for repo");
2507        }
2508    }
2509
2510    Ok(())
2511}
2512
2513/// Phase 4 of tick: unblock parent tasks whose sub-issues are all done.
2514pub(crate) async fn tick_unblock_parents(
2515    backend: &Arc<dyn ExternalBackend>,
2516    task_manager: &Arc<TaskManager>,
2517    store: &Arc<TaskStore>,
2518    repo: &str,
2519) -> anyhow::Result<()> {
2520    let blocked = task_manager.list_all_by_status(Status::Blocked).await?;
2521
2522    // Pre-filter: skip tasks that are blocked for a known reason (e.g. CI failure,
2523    // escalation). These are not waiting on children, so there is no point making
2524    // a GraphQL call for them.
2525    let store_blocked = store
2526        .list_by_status(repo, crate::store::TaskStatus::Blocked)
2527        .await?;
2528    let has_block_reason: std::collections::HashSet<String> = store_blocked
2529        .iter()
2530        .filter(|t| t.block_reason.is_some())
2531        .filter_map(|t| t.external_id.clone())
2532        .collect();
2533
2534    // Spawn a concurrent task for each candidate blocked task.
2535    // Previously these ran sequentially; get_sub_issues is a GraphQL call taking 1-3 s each,
2536    // and with 4 blocked tasks that was 4-12 s of sequential latency every tick.
2537    let handles: Vec<_> = blocked
2538        .into_iter()
2539        .filter(|task| {
2540            if has_block_reason.contains(&task.id.0) {
2541                tracing::debug!(
2542                    task_id = task.id.0,
2543                    "skipping blocked task with known block_reason"
2544                );
2545                false
2546            } else {
2547                true
2548            }
2549        })
2550        .map(|task| {
2551            let backend_clone = Arc::clone(backend);
2552            let task_manager_clone = Arc::clone(task_manager);
2553            let store_clone = Arc::clone(store);
2554            let repo_owned = repo.to_string();
2555            tokio::spawn(async move {
2556                let task_id = &task.id;
2557
2558                let mut children: Vec<ExternalId> = Vec::new();
2559                let mut child_statuses: std::collections::HashMap<
2560                    String,
2561                    crate::store::TaskStatus,
2562                > = std::collections::HashMap::new();
2563
2564                match store_clone.resolve_task_id(&repo_owned, &task_id.0).await {
2565                    Ok(Some(parent_store_id)) => {
2566                        match store_clone.list_children(&repo_owned, parent_store_id).await {
2567                            Ok(store_children) => {
2568                                for child in store_children {
2569                                    let Some(child_external_id) = child.external_id else {
2570                                        continue;
2571                                    };
2572                                    child_statuses.insert(child_external_id.clone(), child.status);
2573                                    children.push(ExternalId(child_external_id));
2574                                }
2575                            }
2576                            Err(e) => {
2577                                tracing::debug!(
2578                                    task_id = task_id.0,
2579                                    ?e,
2580                                    "failed to list store-linked children"
2581                                );
2582                            }
2583                        }
2584                    }
2585                    Ok(None) => {}
2586                    Err(e) => {
2587                        tracing::debug!(
2588                            task_id = task_id.0,
2589                            ?e,
2590                            "failed to resolve blocked task in store for child lookup"
2591                        );
2592                    }
2593                }
2594
2595                if !is_internal_id(&task_id.0) {
2596                    match backend_clone.get_sub_issues(task_id).await {
2597                        Ok(ids) => {
2598                            let mut seen: std::collections::HashSet<String> =
2599                                children.iter().map(|child| child.0.clone()).collect();
2600                            for id in ids {
2601                                if seen.insert(id.0.clone()) {
2602                                    children.push(id);
2603                                }
2604                            }
2605                        }
2606                        Err(e) => {
2607                            tracing::debug!(task_id = task_id.0, ?e, "failed to get sub-issues");
2608                        }
2609                    }
2610                }
2611
2612                // No children means nothing to wait on — skip (may be blocked for other reasons)
2613                if children.is_empty() {
2614                    return;
2615                }
2616
2617                // Check if every child is done using the local store (batched) and
2618                // fall back to the backend only for children not present in the store.
2619                let mut all_done = true;
2620                // Build owned strings for the batch lookup (needed for &str slices below)
2621                let child_ext_strs: Vec<String> =
2622                    children.iter().map(|c| c.0.clone()).collect();
2623                let child_exts: Vec<&str> =
2624                    child_ext_strs.iter().map(std::string::String::as_str).collect();
2625                // Query store for statuses of any children present locally
2626                match store_clone
2627                    .get_statuses_by_external_ids(&repo_owned, &child_exts)
2628                    .await
2629                { Ok(map) => {
2630                    for (k, v) in map {
2631                        child_statuses.insert(k, v);
2632                    }
2633                } _ => {
2634                    tracing::debug!(
2635                        task_id = task_id.0,
2636                        "failed to batch-query store for child statuses; falling back to per-child lookups"
2637                    );
2638                }}
2639
2640                for child_id in &children {
2641                    // Check store first
2642                    if let Some(status) = child_statuses.get(&child_id.0) {
2643                        if *status != crate::store::TaskStatus::Done {
2644                            all_done = false;
2645                            break;
2646                        }
2647                        continue;
2648                    }
2649
2650                    // Not found in store — fall back to backend check
2651                    match backend_clone.get_task(child_id).await {
2652                        Ok(child) => {
2653                            if !child.labels.iter().any(|l| l == Status::Done.as_label()) {
2654                                all_done = false;
2655                                break;
2656                            }
2657                        }
2658                        Err(e) => {
2659                            tracing::debug!(task_id = task_id.0, child = %child_id.0, ?e, "failed to fetch child task from backend — treating as not done");
2660                            all_done = false;
2661                            break;
2662                        }
2663                    }
2664                }
2665
2666                if all_done {
2667                    tracing::info!(
2668                        task_id = task_id.0,
2669                        children = children.len(),
2670                        "all children done, unblocking parent"
2671                    );
2672                    if let Err(e) =
2673                        task_manager_clone.update_task_status(task_id, Status::New).await
2674                    {
2675                        tracing::warn!(task_id = task_id.0, ?e, "failed to unblock parent");
2676                    }
2677                }
2678            })
2679        })
2680        .collect();
2681
2682    for handle in handles {
2683        if let Err(e) = handle.await {
2684            tracing::warn!(?e, "phase 4 sub-issue check panicked");
2685        }
2686    }
2687    Ok(())
2688}
2689
2690/// Phase 5 of tick: run cron job matching and fire any due jobs.
2691pub(crate) async fn tick_job_scheduler(
2692    jobs_path: &std::path::Path,
2693    backend: &Arc<dyn ExternalBackend>,
2694    store: Option<&Arc<TaskStore>>,
2695    repo: &str,
2696    transport: Option<&Arc<crate::channels::transport::Transport>>,
2697) -> anyhow::Result<()> {
2698    jobs::tick(jobs_path, backend, store, repo, transport).await
2699}
2700
2701/// Core tick for one project, up to the routing phase.
2702///
2703/// Every phase here is scoped to one project, so all projects can run this at the same time:
2704/// 1. `tick_check_session_completions` — poll tmux for finished sessions
2705/// 2. `tick_detect_silent_agents`      — cooldown agents that produced no output
2706/// 3. `tick_recover_stuck_tasks`       — reset in_progress/in_review tasks with no session
2707///
2708/// The store-wide work that used to sit in here runs once per tick in `tick_all`:
2709/// `tick_global_sweeps` and `reconcile_dependencies`.
2710#[expect(
2711    clippy::too_many_arguments,
2712    reason = "each argument is engine state the tick passes down"
2713)]
2714pub(crate) async fn tick_before_routing(
2715    backend: &Arc<dyn ExternalBackend>,
2716    tmux: &Arc<TmuxManager>,
2717    repo: &str,
2718    capture: &Arc<CaptureService>,
2719    config: &EngineConfig,
2720    task_manager: &Arc<TaskManager>,
2721    dispatching: &Arc<DashMap<String, String>>,
2722    store: &Arc<TaskStore>,
2723    session_map: &std::collections::HashMap<String, bool>,
2724) {
2725    let mut phases: Vec<(&str, u128)> = Vec::new();
2726    let mut at = Instant::now();
2727
2728    if let Err(e) = tick_check_session_completions(tmux, repo, capture, store).await {
2729        tracing::warn!(err = %e, "session completion phase failed");
2730    }
2731    phases.push(("sessions", at.elapsed().as_millis()));
2732
2733    // Every node tends its own sessions, only the leader decides. Followers dispatch once
2734    // tasks are assigned to nodes
2735    if !store.is_leader() {
2736        super::log_phase_timings("tick.pre_routing", repo, &phases);
2737        return;
2738    }
2739
2740    at = Instant::now();
2741    if let Err(e) =
2742        tick_detect_silent_agents(tmux, repo, capture, backend, task_manager, config, store).await
2743    {
2744        tracing::warn!(err = %e, "silent-agent detection phase failed");
2745    }
2746    phases.push(("silent_agents", at.elapsed().as_millis()));
2747
2748    at = Instant::now();
2749    if let Err(e) = tick_recover_stuck_tasks(
2750        backend,
2751        tmux,
2752        repo,
2753        task_manager,
2754        config,
2755        store,
2756        session_map,
2757        dispatching,
2758    )
2759    .await
2760    {
2761        tracing::warn!(err = %e, "stuck-task recovery phase failed");
2762    }
2763    phases.push(("stuck_tasks", at.elapsed().as_millis()));
2764
2765    super::log_phase_timings("tick.pre_routing", repo, &phases);
2766}
2767
2768/// Core tick for one project, after the routing phase.
2769///
2770/// Takes the router's decision as a `DispatchMode` value instead of the router itself, so
2771/// every project can run this at the same time:
2772/// 1. `tick_dispatch_tasks`  — spawn agents for status:routed tasks
2773/// 2. `tick_unblock_parents` — unblock parents whose sub-issues are all done
2774/// 3. `tick_job_scheduler`   — fire due cron jobs
2775#[expect(
2776    clippy::too_many_arguments,
2777    reason = "each argument is engine state the tick passes down"
2778)]
2779pub(crate) async fn tick_after_routing(
2780    backend: &Arc<dyn ExternalBackend>,
2781    tmux: &Arc<TmuxManager>,
2782    repo: &str,
2783    runner: &Arc<TaskRunner>,
2784    capture: &Arc<CaptureService>,
2785    semaphore: &Arc<Semaphore>,
2786    project_dir: &std::path::Path,
2787    task_manager: &Arc<TaskManager>,
2788    weight_tx: &mpsc::Sender<WeightSignal>,
2789    dispatch_mode: DispatchMode,
2790    dispatching: &Arc<DashMap<String, String>>,
2791    store: &Arc<TaskStore>,
2792    transport: Option<&Arc<crate::channels::transport::Transport>>,
2793    session_map: &std::collections::HashMap<String, bool>,
2794) {
2795    let mut phases: Vec<(&str, u128)> = Vec::new();
2796    let mut at = Instant::now();
2797
2798    if let Err(e) = tick_dispatch_tasks(
2799        backend,
2800        tmux,
2801        repo,
2802        runner,
2803        capture,
2804        semaphore,
2805        task_manager,
2806        weight_tx,
2807        dispatch_mode,
2808        dispatching,
2809        store,
2810        session_map,
2811    )
2812    .await
2813    {
2814        tracing::warn!(err = %e, "dispatch phase failed");
2815    }
2816    phases.push(("dispatch", at.elapsed().as_millis()));
2817
2818    at = Instant::now();
2819    if let Err(e) = tick_unblock_parents(backend, task_manager, store, repo).await {
2820        tracing::warn!(err = %e, "parent-unblock phase failed");
2821    }
2822    phases.push(("unblock_parents", at.elapsed().as_millis()));
2823
2824    at = Instant::now();
2825    let jobs_path = project_dir.join(".orch.yml");
2826    if let Err(e) = tick_job_scheduler(&jobs_path, backend, Some(store), repo, transport).await {
2827        tracing::error!(?e, "job scheduler tick failed");
2828    }
2829    phases.push(("jobs", at.elapsed().as_millis()));
2830
2831    super::log_phase_timings("tick.post_routing", repo, &phases);
2832}
2833
2834/// Core tick — runs every 10s for every project.
2835///
2836/// Projects no longer wait on each other: the phases around routing run for all of them at
2837/// once, so tick time tracks the slowest project instead of the sum. Routing stays serial
2838/// because it needs the router write lock and one shared `RoutingTickQuota`, which is what
2839/// gives `router.max_tasks_per_tick` its meaning.
2840///
2841/// The tmux session map, the global sweeps and the dependency reconcile are the same for
2842/// every project, so they run once per tick rather than once per project.
2843#[expect(
2844    clippy::too_many_arguments,
2845    reason = "each argument is engine state the tick passes down"
2846)]
2847pub(crate) async fn tick_all(
2848    engines: &[ProjectEngine],
2849    tmux: &Arc<TmuxManager>,
2850    capture: &Arc<CaptureService>,
2851    semaphore: &Arc<Semaphore>,
2852    config: &EngineConfig,
2853    router: &Arc<tokio::sync::RwLock<Router>>,
2854    weight_tx: &mpsc::Sender<WeightSignal>,
2855    dispatching: &Arc<DashMap<String, String>>,
2856    transport: Option<&Arc<crate::channels::transport::Transport>>,
2857    active_repos: &std::collections::HashSet<String>,
2858    auto_merge_in_flight: &Arc<DashSet<String>>,
2859) {
2860    let _tick_span = tracing::info_span!("engine.tick").entered();
2861
2862    let Some(first) = engines.first() else {
2863        return;
2864    };
2865    // All project engines share one SQLite store, so leadership and the global sweeps below
2866    // are the same whichever project they are read from
2867    let store = &first.store;
2868
2869    // Calls made since the previous tick, so hourly quota exhaustion can be traced
2870    for (project, call_type, calls) in crate::github::http::drain_call_counts()
2871        .into_iter()
2872        .take(10)
2873    {
2874        tracing::debug!(
2875            project,
2876            call_type,
2877            calls,
2878            "github api calls since last tick"
2879        );
2880    }
2881
2882    // One `tmux list-panes -a` for every project's stuck-task recovery, instead of one each
2883    let session_map = tmux.batch_session_active().await;
2884
2885    let mut passes = Vec::with_capacity(engines.len());
2886    for engine in engines {
2887        passes.push(REPO_CONTEXT.scope(
2888            engine.repo.clone(),
2889            tick_before_routing(
2890                &engine.backend,
2891                tmux,
2892                &engine.repo,
2893                capture,
2894                config,
2895                &engine.task_manager,
2896                dispatching,
2897                &engine.store,
2898                &session_map,
2899            ),
2900        ));
2901    }
2902    futures::future::join_all(passes).await;
2903
2904    if !store.is_leader() {
2905        return;
2906    }
2907
2908    // Store-wide, so once per tick rather than once per project. Both read every repo, and
2909    // the NeedsReview sweep increments a counter it just read, which concurrent per-project
2910    // passes would each consume (#3864)
2911    let at = Instant::now();
2912    {
2913        let router_guard = router.read().await;
2914        tick_global_sweeps(
2915            &first.task_manager,
2916            store,
2917            &router_guard,
2918            dispatching,
2919            auto_merge_in_flight,
2920        )
2921        .await;
2922    }
2923    let sweeps_ms = at.elapsed().as_millis();
2924
2925    let at = Instant::now();
2926    recover_untracked_repo_stuck_tasks(
2927        tmux,
2928        config,
2929        store,
2930        &session_map,
2931        dispatching,
2932        active_repos,
2933    )
2934    .await;
2935    let untracked_ms = at.elapsed().as_millis();
2936
2937    let at = Instant::now();
2938    if let Err(err) = crate::engine::dependencies::reconcile_dependencies(
2939        first.backend.as_ref(),
2940        store,
2941        transport.map(std::convert::AsRef::as_ref),
2942    )
2943    .await
2944    {
2945        tracing::warn!(err = %err, "dependency reconcile failed");
2946    }
2947    super::log_phase_timings(
2948        "tick.global",
2949        &first.repo,
2950        &[
2951            ("global_sweeps", sweeps_ms),
2952            ("untracked_stuck", untracked_ms),
2953            ("dependencies", at.elapsed().as_millis()),
2954        ],
2955    );
2956
2957    {
2958        let mut router_guard = router.write().await;
2959        let mut routing_quota =
2960            RoutingTickQuota::new(crate::engine::router::config::max_tasks_per_routing_tick());
2961        for engine in engines {
2962            let at = Instant::now();
2963            REPO_CONTEXT
2964                .scope(engine.repo.clone(), async {
2965                    if let Err(e) = tick_route_tasks(
2966                        &engine.backend,
2967                        &engine.task_manager,
2968                        &mut router_guard,
2969                        &engine.store,
2970                        &engine.repo,
2971                        &mut routing_quota,
2972                    )
2973                    .await
2974                    {
2975                        tracing::warn!(err = %e, "routing phase failed");
2976                    }
2977                })
2978                .await;
2979            super::log_phase_timings(
2980                "tick.routing",
2981                &engine.repo,
2982                &[("route", at.elapsed().as_millis())],
2983            );
2984        }
2985
2986        // Global routing sweep for tasks from inactive or removed repos.
2987        // tick_route_tasks is scoped to the active repo; tasks from projects no longer
2988        // in config are never returned by list_routable and would stay in 'new' indefinitely.
2989        if let Err(e) =
2990            route_new_tasks_global(&mut router_guard, store, active_repos, &mut routing_quota).await
2991        {
2992            tracing::warn!(err = %e, "global new-task routing sweep failed");
2993        }
2994    }
2995
2996    let dispatch_mode = dispatch_mode_from_router(&*router.read().await);
2997
2998    // Re-read the session map: the completion phase above kills the sessions of finished
2999    // agents, and dispatch must not treat a killed session as one that still blocks a task
3000    let session_map = tmux.batch_session_active().await;
3001
3002    let mut passes = Vec::with_capacity(engines.len());
3003    for engine in engines {
3004        passes.push(REPO_CONTEXT.scope(
3005            engine.repo.clone(),
3006            tick_after_routing(
3007                &engine.backend,
3008                tmux,
3009                &engine.repo,
3010                &engine.runner,
3011                capture,
3012                semaphore,
3013                &engine.project_dir,
3014                &engine.task_manager,
3015                weight_tx,
3016                dispatch_mode,
3017                dispatching,
3018                &engine.store,
3019                transport,
3020                &session_map,
3021            ),
3022        ));
3023    }
3024    futures::future::join_all(passes).await;
3025
3026    // Global dispatch sweep for routed tasks from inactive or removed repos.
3027    // tick_dispatch_tasks is scoped to each active repo; tasks from projects no longer
3028    // in config are never returned by list_external_by_status and would stay in
3029    // 'routed' indefinitely (see #3413). Excludes the full active-repo set, not just
3030    // one repo, so it doesn't treat other active repos as orphaned.
3031    if let Err(e) = dispatch_routed_tasks_global(
3032        tmux,
3033        capture,
3034        semaphore,
3035        weight_tx,
3036        dispatch_mode,
3037        dispatching,
3038        store,
3039        &session_map,
3040        active_repos,
3041    )
3042    .await
3043    {
3044        tracing::warn!(err = %e, "global routed-task dispatch sweep failed");
3045    }
3046}
3047
3048#[cfg(test)]
3049mod tests {
3050    use super::*;
3051    use crate::backends::{ExternalId, ExternalTask, Mention, Status};
3052    use crate::channels::transport::Transport;
3053    use crate::engine::tasks::TaskManager;
3054    use crate::store::TaskStore;
3055    use crate::tmux::{SessionGuard, TmuxManager};
3056    use async_trait::async_trait;
3057    use std::sync::{Arc, Mutex};
3058
3059    #[test]
3060    fn skips_agent_for_human_and_no_agent_labels() {
3061        assert!(skips_agent(&["owner:human".to_string()]));
3062        assert!(skips_agent(&["no-agent".to_string()]));
3063        assert!(!skips_agent(&["status:new".to_string()]));
3064    }
3065
3066    /// A router reporting one available agent with no healthy "review" model, so the
3067    /// global NeedsReview refire sweep in `tick_global_sweeps` treats every review agent
3068    /// as cooled and skips — keeping it a no-op for tests unrelated to it.
3069    fn cooled_review_router_for_test() -> Router {
3070        Router::new_for_test(
3071            crate::engine::router::RouterConfig::default(),
3072            vec!["claude".to_string()],
3073        )
3074    }
3075
3076    // ── minimal mock backend ─────────────────────────────────────────────────
3077
3078    /// Configurable mock backend for tick tests.
3079    struct MockBackend {
3080        /// Tasks returned by `list_by_status(Blocked)`.
3081        blocked_tasks: Vec<ExternalTask>,
3082        /// Sub-issues map: task_id → list of child ExternalIds.
3083        sub_issues: std::collections::HashMap<String, Vec<ExternalId>>,
3084        /// Tasks returned by `get_task`. Keyed by id.0.
3085        tasks_by_id: std::collections::HashMap<String, ExternalTask>,
3086        /// Recorded `get_sub_issues` calls.
3087        get_sub_issues_calls: Arc<Mutex<Vec<String>>>,
3088        /// Recorded `update_status` calls.
3089        status_updates: Arc<Mutex<Vec<(String, Status)>>>,
3090        /// Recorded `remove_label` calls.
3091        removed_labels: Arc<Mutex<Vec<(String, String)>>>,
3092        /// Recorded `post_comment` calls.
3093        posted_comments: Arc<Mutex<Vec<(String, String)>>>,
3094    }
3095
3096    impl MockBackend {
3097        fn new() -> Self {
3098            Self {
3099                blocked_tasks: vec![],
3100                sub_issues: std::collections::HashMap::new(),
3101                tasks_by_id: std::collections::HashMap::new(),
3102                get_sub_issues_calls: Arc::new(Mutex::new(vec![])),
3103                status_updates: Arc::new(Mutex::new(vec![])),
3104                removed_labels: Arc::new(Mutex::new(vec![])),
3105                posted_comments: Arc::new(Mutex::new(vec![])),
3106            }
3107        }
3108    }
3109
3110    fn make_task(id: &str, labels: &[&str]) -> ExternalTask {
3111        ExternalTask {
3112            id: ExternalId(id.to_string()),
3113            title: format!("Task {id}"),
3114            body: String::new(),
3115            state: "open".to_string(),
3116            labels: labels
3117                .iter()
3118                .map(std::string::ToString::to_string)
3119                .collect(),
3120            author: "bot".to_string(),
3121            created_at: "2026-01-01T00:00:00Z".to_string(),
3122            updated_at: "2026-01-01T00:00:00Z".to_string(),
3123            url: format!("https://github.com/test/test/issues/{id}"),
3124        }
3125    }
3126
3127    #[async_trait]
3128    impl ExternalBackend for MockBackend {
3129        fn name(&self) -> &'static str {
3130            "mock"
3131        }
3132        async fn create_task(
3133            &self,
3134            _title: &str,
3135            _body: &str,
3136            _labels: &[String],
3137        ) -> anyhow::Result<ExternalId> {
3138            Ok(ExternalId("new".to_string()))
3139        }
3140        async fn get_task(&self, id: &ExternalId) -> anyhow::Result<ExternalTask> {
3141            self.tasks_by_id
3142                .get(&id.0)
3143                .cloned()
3144                .ok_or_else(|| anyhow::anyhow!("task not found: {}", id.0))
3145        }
3146        async fn list_by_status(&self, status: Status) -> anyhow::Result<Vec<ExternalTask>> {
3147            if status == Status::Blocked {
3148                Ok(self.blocked_tasks.clone())
3149            } else {
3150                Ok(vec![])
3151            }
3152        }
3153        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
3154            Ok(vec![])
3155        }
3156        async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
3157            self.posted_comments
3158                .lock()
3159                .unwrap()
3160                .push((_id.0.clone(), _body.to_string()));
3161            Ok(())
3162        }
3163        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
3164            Ok(())
3165        }
3166        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
3167            self.removed_labels
3168                .lock()
3169                .unwrap()
3170                .push((_id.0.clone(), _label.to_string()));
3171            Ok(())
3172        }
3173        async fn get_sub_issues(&self, id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
3174            self.get_sub_issues_calls.lock().unwrap().push(id.0.clone());
3175            Ok(self.sub_issues.get(&id.0).cloned().unwrap_or_default())
3176        }
3177        async fn create_sub_task(
3178            &self,
3179            _parent: &ExternalId,
3180            _title: &str,
3181            _body: &str,
3182            _labels: &[String],
3183        ) -> anyhow::Result<ExternalId> {
3184            Ok(ExternalId("child".to_string()))
3185        }
3186        async fn ensure_status_label(&self, _label: &str) -> anyhow::Result<()> {
3187            Ok(())
3188        }
3189        async fn has_open_issue_with_title(
3190            &self,
3191            _title: &str,
3192            _label: &str,
3193        ) -> anyhow::Result<bool> {
3194            Ok(false)
3195        }
3196        async fn health_check(&self) -> anyhow::Result<()> {
3197            Ok(())
3198        }
3199        async fn is_pr_merged(&self, _branch: &str) -> anyhow::Result<bool> {
3200            Ok(false)
3201        }
3202        async fn get_authenticated_user(&self) -> anyhow::Result<Option<String>> {
3203            Ok(Some("testbot".to_string()))
3204        }
3205        async fn get_mentions(&self, _since: &str) -> anyhow::Result<Vec<Mention>> {
3206            Ok(vec![])
3207        }
3208        async fn update_status(&self, id: &ExternalId, status: Status) -> anyhow::Result<()> {
3209            self.status_updates
3210                .lock()
3211                .unwrap()
3212                .push((id.0.clone(), status));
3213            Ok(())
3214        }
3215    }
3216
3217    fn make_task_manager(backend: Arc<dyn ExternalBackend>) -> Arc<TaskManager> {
3218        Arc::new(TaskManager::new(backend))
3219    }
3220
3221    // ── tick_unblock_parents ─────────────────────────────────────────────────
3222
3223    #[tokio::test]
3224    async fn unblock_parents_unblocks_when_all_children_done() {
3225        let mut mock = MockBackend::new();
3226
3227        // Blocked parent with two done children
3228        let parent = make_task("10", &["status:blocked"]);
3229        mock.blocked_tasks.push(parent.clone());
3230        mock.sub_issues.insert(
3231            "10".to_string(),
3232            vec![ExternalId("11".to_string()), ExternalId("12".to_string())],
3233        );
3234        mock.tasks_by_id
3235            .insert("11".to_string(), make_task("11", &["status:done"]));
3236        mock.tasks_by_id
3237            .insert("12".to_string(), make_task("12", &["status:done"]));
3238
3239        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3240        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3241        let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
3242        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3243
3244        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3245            .await
3246            .unwrap();
3247
3248        let updates = status_updates.lock().unwrap();
3249        assert_eq!(updates.len(), 1, "parent should be unblocked");
3250        assert_eq!(updates[0], ("10".to_string(), Status::New));
3251    }
3252
3253    #[tokio::test]
3254    async fn unblock_parents_skips_when_child_not_done() {
3255        let mut mock = MockBackend::new();
3256
3257        let parent = make_task("20", &["status:blocked"]);
3258        mock.blocked_tasks.push(parent.clone());
3259        mock.sub_issues.insert(
3260            "20".to_string(),
3261            vec![ExternalId("21".to_string()), ExternalId("22".to_string())],
3262        );
3263        // Child 21 is done, child 22 is still in_progress
3264        mock.tasks_by_id
3265            .insert("21".to_string(), make_task("21", &["status:done"]));
3266        mock.tasks_by_id
3267            .insert("22".to_string(), make_task("22", &["status:in_progress"]));
3268
3269        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3270        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3271        let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
3272        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3273
3274        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3275            .await
3276            .unwrap();
3277
3278        let updates = status_updates.lock().unwrap();
3279        assert!(
3280            updates.is_empty(),
3281            "should not unblock when a child is still running"
3282        );
3283    }
3284
3285    #[tokio::test]
3286    async fn unblock_parents_skips_task_with_no_children() {
3287        let mut mock = MockBackend::new();
3288
3289        // Blocked task with no sub-issues (blocked for a different reason)
3290        let parent = make_task("30", &["status:blocked"]);
3291        mock.blocked_tasks.push(parent.clone());
3292        // No entry in sub_issues → get_sub_issues returns []
3293
3294        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3295        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3296        let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
3297        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3298
3299        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3300            .await
3301            .unwrap();
3302
3303        let updates = status_updates.lock().unwrap();
3304        assert!(
3305            updates.is_empty(),
3306            "should not unblock tasks with no sub-issues"
3307        );
3308    }
3309
3310    #[tokio::test]
3311    async fn unblock_parents_skips_task_with_block_reason_in_store() {
3312        // A blocked task that has a known block_reason in the store should never
3313        // trigger a get_sub_issues GraphQL call — it is not waiting on children.
3314        let mut mock = MockBackend::new();
3315
3316        // Blocked parent in the backend list
3317        let parent = make_task("60", &["status:blocked"]);
3318        mock.blocked_tasks.push(parent.clone());
3319        // Give it children in the backend so that, if the skip logic is wrong,
3320        // the parent would get unblocked.
3321        mock.sub_issues
3322            .insert("60".to_string(), vec![ExternalId("61".to_string())]);
3323        mock.tasks_by_id
3324            .insert("61".to_string(), make_task("61", &["status:done"]));
3325
3326        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3327        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3328        let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
3329
3330        // Insert the blocked task into the store with a block_reason set.
3331        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3332        let task_id = store
3333            .upsert_external(&crate::store::UpsertExternal {
3334                repo: "test/repo",
3335                ext_id: "60",
3336                title: "Task 60",
3337                body: "",
3338                author: "bot",
3339                url: "https://github.com/test/test/issues/60",
3340                labels: &[],
3341                origin: "github",
3342            })
3343            .await
3344            .unwrap();
3345        store
3346            .update_status(task_id, crate::store::TaskStatus::Blocked)
3347            .await
3348            .unwrap();
3349        store
3350            .set_block_reason(task_id, Some("CI failure limit reached"))
3351            .await
3352            .unwrap();
3353
3354        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3355            .await
3356            .unwrap();
3357
3358        let updates = status_updates.lock().unwrap();
3359        assert!(
3360            updates.is_empty(),
3361            "task with block_reason should be skipped — no API call, no unblock"
3362        );
3363    }
3364
3365    #[tokio::test]
3366    async fn unblock_parents_unblocks_internal_parent_from_store_children() {
3367        let mock = MockBackend::new();
3368        let get_sub_issues_calls = std::sync::Arc::clone(&mock.get_sub_issues_calls);
3369        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3370        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3371        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3372        let task_manager = Arc::new(TaskManager::with_store(
3373            std::sync::Arc::clone(&backend),
3374            std::sync::Arc::clone(&store),
3375            "test/repo".to_string(),
3376        ));
3377
3378        let parent_id = store
3379            .create_internal("test/repo", "Parent", "", "job", "daily", None)
3380            .await
3381            .unwrap();
3382        store
3383            .update_status(parent_id, crate::store::TaskStatus::Blocked)
3384            .await
3385            .unwrap();
3386        let child_id = store
3387            .create(&crate::store::NewTask {
3388                external_id: Some("91".to_string()),
3389                repo: "test/repo".to_string(),
3390                origin: "github".to_string(),
3391                title: "Child".to_string(),
3392                body: String::new(),
3393                source: String::new(),
3394                source_id: String::new(),
3395                author: String::new(),
3396                url: String::new(),
3397                labels: vec![],
3398                parent_id: Some(parent_id),
3399            })
3400            .await
3401            .unwrap();
3402        store
3403            .update_status(child_id, crate::store::TaskStatus::Done)
3404            .await
3405            .unwrap();
3406
3407        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
3408            .await
3409            .unwrap();
3410
3411        let parent = store.get(parent_id).await.unwrap();
3412        assert_eq!(parent.status, crate::store::TaskStatus::New);
3413        assert!(
3414            status_updates.lock().unwrap().is_empty(),
3415            "internal parent should be updated in store without backend mirroring"
3416        );
3417        assert!(
3418            get_sub_issues_calls.lock().unwrap().is_empty(),
3419            "internal parent should not query backend sub-issues"
3420        );
3421    }
3422
3423    // ── tick_recover_stuck_tasks (InReview) ──────────────────────────────────
3424
3425    /// Set `updated_at` to a far-past date for a task in an in-memory store.
3426    #[cfg(test)]
3427    async fn set_task_updated_at_past(store: &TaskStore, task_id: i64) {
3428        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
3429            .bind(task_id)
3430            .execute(store.pool())
3431            .await
3432            .unwrap();
3433    }
3434
3435    /// Age the in_review status-entry the same way the stale sweep tests do
3436    async fn set_in_review_entered_at_past(store: &TaskStore, task_id: i64) {
3437        sqlx::query(
3438            "UPDATE task_activity SET timestamp = '2020-01-01T00:00:00Z'
3439             WHERE task_id = ? AND to_status = 'in_review'",
3440        )
3441        .bind(task_id)
3442        .execute(store.pool())
3443        .await
3444        .unwrap();
3445    }
3446
3447    #[tokio::test]
3448    async fn recover_stuck_tasks_resets_external_in_review_to_needs_review() {
3449        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3450        let mock = MockBackend::new();
3451        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3452        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3453        let task_manager = Arc::new(TaskManager::with_store(
3454            std::sync::Arc::clone(&backend),
3455            std::sync::Arc::clone(&store),
3456            "owner/repo".to_string(),
3457        ));
3458        let tmux = Arc::new(TmuxManager::new());
3459        let config = EngineConfig {
3460            no_session_stuck_timeout: 600,
3461            stuck_timeout: 1800,
3462            ..EngineConfig::default()
3463        };
3464
3465        // Insert an external task and age both the status entry and updated_at
3466        let id = store
3467            .upsert_external(&crate::store::UpsertExternal {
3468                repo: "owner/repo",
3469                ext_id: "99",
3470                title: "Stuck InReview",
3471                body: "",
3472                author: "",
3473                url: "",
3474                labels: &[],
3475                origin: "github",
3476            })
3477            .await
3478            .unwrap();
3479        store
3480            .update_status(id, crate::store::TaskStatus::InReview)
3481            .await
3482            .unwrap();
3483        set_in_review_entered_at_past(&store, id).await;
3484        set_task_updated_at_past(&store, id).await;
3485
3486        tick_recover_stuck_tasks(
3487            &backend,
3488            &tmux,
3489            "owner/repo",
3490            &task_manager,
3491            &config,
3492            &store,
3493            &std::collections::HashMap::new(),
3494            &Arc::new(DashMap::new()),
3495        )
3496        .await
3497        .unwrap();
3498
3499        let updates = status_updates.lock().unwrap();
3500        assert_eq!(updates.len(), 1, "stuck InReview task should be recovered");
3501        assert_eq!(updates[0].0, "99");
3502        assert_eq!(updates[0].1, Status::NeedsReview);
3503    }
3504
3505    #[tokio::test]
3506    async fn recover_stuck_tasks_skips_recent_in_review() {
3507        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3508        let mock = MockBackend::new();
3509        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3510        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3511        let task_manager = Arc::new(TaskManager::with_store(
3512            std::sync::Arc::clone(&backend),
3513            std::sync::Arc::clone(&store),
3514            "owner/repo".to_string(),
3515        ));
3516        let tmux = Arc::new(TmuxManager::new());
3517        let config = EngineConfig {
3518            no_session_stuck_timeout: 600, // 10 minutes — recent task won't trigger
3519            stuck_timeout: 1800,
3520            ..EngineConfig::default()
3521        };
3522
3523        // Age updated_at, leave the in_review status entry at now
3524        let id = store
3525            .upsert_external(&crate::store::UpsertExternal {
3526                repo: "owner/repo",
3527                ext_id: "100",
3528                title: "Recent InReview",
3529                body: "",
3530                author: "",
3531                url: "",
3532                labels: &[],
3533                origin: "github",
3534            })
3535            .await
3536            .unwrap();
3537        store
3538            .update_status(id, crate::store::TaskStatus::InReview)
3539            .await
3540            .unwrap();
3541        set_task_updated_at_past(&store, id).await;
3542
3543        tick_recover_stuck_tasks(
3544            &backend,
3545            &tmux,
3546            "owner/repo",
3547            &task_manager,
3548            &config,
3549            &store,
3550            &std::collections::HashMap::new(),
3551            &Arc::new(DashMap::new()),
3552        )
3553        .await
3554        .unwrap();
3555
3556        let updates = status_updates.lock().unwrap();
3557        assert!(
3558            updates.is_empty(),
3559            "recent InReview task should not be recovered"
3560        );
3561    }
3562
3563    #[tokio::test]
3564    async fn recover_stuck_tasks_resets_internal_in_review_to_needs_review() {
3565        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3566        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3567        let task_manager = Arc::new(TaskManager::with_store(
3568            std::sync::Arc::clone(&backend),
3569            std::sync::Arc::clone(&store),
3570            "owner/repo".to_string(),
3571        ));
3572        let tmux = Arc::new(TmuxManager::new());
3573        let config = EngineConfig {
3574            no_session_stuck_timeout: 600,
3575            stuck_timeout: 1800,
3576            ..EngineConfig::default()
3577        };
3578
3579        // Create an internal task in InReview and age both clocks
3580        let id = store
3581            .create_internal("owner/repo", "Internal InReview", "", "cron", "1", None)
3582            .await
3583            .unwrap();
3584        store
3585            .update_status(id, crate::store::TaskStatus::InReview)
3586            .await
3587            .unwrap();
3588        set_in_review_entered_at_past(&store, id).await;
3589        set_task_updated_at_past(&store, id).await;
3590
3591        tick_recover_stuck_tasks(
3592            &backend,
3593            &tmux,
3594            "owner/repo",
3595            &task_manager,
3596            &config,
3597            &store,
3598            &std::collections::HashMap::new(),
3599            &Arc::new(DashMap::new()),
3600        )
3601        .await
3602        .unwrap();
3603
3604        let task = store.get(id).await.unwrap();
3605        assert_eq!(
3606            task.status,
3607            crate::store::TaskStatus::NeedsReview,
3608            "stuck internal InReview task should be reset to NeedsReview"
3609        );
3610    }
3611
3612    #[tokio::test]
3613    async fn recover_stuck_tasks_ages_in_review_from_status_entry_not_updated_at() {
3614        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3615        let mock = MockBackend::new();
3616        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3617        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3618        let task_manager = Arc::new(TaskManager::with_store(
3619            std::sync::Arc::clone(&backend),
3620            std::sync::Arc::clone(&store),
3621            "owner/repo".to_string(),
3622        ));
3623        let tmux = Arc::new(TmuxManager::new());
3624        let config = EngineConfig {
3625            no_session_stuck_timeout: 600,
3626            stuck_timeout: 1800,
3627            ..EngineConfig::default()
3628        };
3629
3630        let id = store
3631            .upsert_external(&crate::store::UpsertExternal {
3632                repo: "owner/repo",
3633                ext_id: "102",
3634                title: "Fresh updated_at InReview",
3635                body: "",
3636                author: "",
3637                url: "",
3638                labels: &[],
3639                origin: "github",
3640            })
3641            .await
3642            .unwrap();
3643        store
3644            .update_status(id, crate::store::TaskStatus::InReview)
3645            .await
3646            .unwrap();
3647        set_in_review_entered_at_past(&store, id).await;
3648        store
3649            .set_fields(id, &[("pr_number", serde_json::json!(42))])
3650            .await
3651            .unwrap();
3652
3653        tick_recover_stuck_tasks(
3654            &backend,
3655            &tmux,
3656            "owner/repo",
3657            &task_manager,
3658            &config,
3659            &store,
3660            &std::collections::HashMap::new(),
3661            &Arc::new(DashMap::new()),
3662        )
3663        .await
3664        .unwrap();
3665
3666        let updates = status_updates.lock().unwrap();
3667        assert_eq!(
3668            updates.len(),
3669            1,
3670            "in_review stuck recovery must age from the status entry even when updated_at is fresh"
3671        );
3672        assert_eq!(updates[0].0, "102");
3673        assert_eq!(updates[0].1, Status::NeedsReview);
3674    }
3675
3676    /// An in_review task whose review session is gone goes back to needs_review, unless its
3677    /// review already recorded a decision and the task is only waiting on the merge
3678    async fn orphaned_review_status_after_recovery(
3679        with_decision: bool,
3680    ) -> crate::store::TaskStatus {
3681        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3682        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3683        let task_manager = Arc::new(TaskManager::with_store(
3684            std::sync::Arc::clone(&backend),
3685            std::sync::Arc::clone(&store),
3686            "owner/repo".to_string(),
3687        ));
3688        let tmux = Arc::new(TmuxManager::new());
3689
3690        let id = store
3691            .upsert_external(&crate::store::UpsertExternal {
3692                repo: "owner/repo",
3693                ext_id: "55",
3694                title: "Orphaned review",
3695                body: "",
3696                author: "",
3697                url: "",
3698                labels: &[],
3699                origin: "github",
3700            })
3701            .await
3702            .unwrap();
3703        store
3704            .update_status(id, crate::store::TaskStatus::InReview)
3705            .await
3706            .unwrap();
3707        // Entered in_review long ago. updated_at stays fresh, as review_poll rewrites it
3708        set_in_review_entered_at_past(&store, id).await;
3709        if with_decision {
3710            crate::store::store_log_activity(
3711                &Some(std::sync::Arc::clone(&store)),
3712                "owner/repo",
3713                "55",
3714                "review_decision",
3715                None,
3716                None,
3717                None::<&str>,
3718                None::<&str>,
3719                Some(&serde_json::json!({"decision": "approve"})),
3720            )
3721            .await;
3722        }
3723
3724        tick_recover_stuck_tasks(
3725            &backend,
3726            &tmux,
3727            "owner/repo",
3728            &task_manager,
3729            &EngineConfig::default(),
3730            &store,
3731            &std::collections::HashMap::new(),
3732            &Arc::new(DashMap::new()),
3733        )
3734        .await
3735        .unwrap();
3736
3737        store.get(id).await.unwrap().status
3738    }
3739
3740    #[tokio::test]
3741    async fn stale_in_review_recovery_resets_orphaned_task_without_session() {
3742        assert_eq!(
3743            orphaned_review_status_after_recovery(false).await,
3744            crate::store::TaskStatus::NeedsReview
3745        );
3746    }
3747
3748    #[tokio::test]
3749    async fn stale_in_review_recovery_leaves_decided_task_waiting_on_merge() {
3750        assert_eq!(
3751            orphaned_review_status_after_recovery(true).await,
3752            crate::store::TaskStatus::InReview
3753        );
3754    }
3755
3756    #[tokio::test]
3757    async fn recover_stuck_tasks_resets_review_agent_failures() {
3758        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3759        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
3760        let task_manager = Arc::new(TaskManager::with_store(
3761            std::sync::Arc::clone(&backend),
3762            std::sync::Arc::clone(&store),
3763            "owner/repo".to_string(),
3764        ));
3765        let tmux = Arc::new(TmuxManager::new());
3766
3767        let id = store
3768            .upsert_external(&crate::store::UpsertExternal {
3769                repo: "owner/repo",
3770                ext_id: "104",
3771                title: "Orphaned review with failures",
3772                body: "",
3773                author: "",
3774                url: "",
3775                labels: &[],
3776                origin: "github",
3777            })
3778            .await
3779            .unwrap();
3780        store
3781            .update_status(id, crate::store::TaskStatus::InReview)
3782            .await
3783            .unwrap();
3784        store
3785            .set_fields(id, &[("review_agent_failures", serde_json::json!(2))])
3786            .await
3787            .unwrap();
3788        set_in_review_entered_at_past(&store, id).await;
3789
3790        tick_recover_stuck_tasks(
3791            &backend,
3792            &tmux,
3793            "owner/repo",
3794            &task_manager,
3795            &EngineConfig::default(),
3796            &store,
3797            &std::collections::HashMap::new(),
3798            &Arc::new(DashMap::new()),
3799        )
3800        .await
3801        .unwrap();
3802
3803        let task = store.get(id).await.unwrap();
3804        assert_eq!(task.status, crate::store::TaskStatus::NeedsReview);
3805        assert_eq!(
3806            task.review_agent_failures, 0,
3807            "a dead review session is an infrastructure event, not an agent failure"
3808        );
3809    }
3810
3811    #[tokio::test]
3812    async fn recover_stuck_tasks_reclaims_hung_in_review_session() {
3813        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3814        let mock = MockBackend::new();
3815        let status_updates = std::sync::Arc::clone(&mock.status_updates);
3816        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3817        let task_manager = Arc::new(TaskManager::with_store(
3818            std::sync::Arc::clone(&backend),
3819            std::sync::Arc::clone(&store),
3820            "owner/repo".to_string(),
3821        ));
3822        let tmux = Arc::new(TmuxManager::new());
3823        let config = EngineConfig {
3824            no_session_stuck_timeout: 600,
3825            stuck_timeout: 1800,
3826            ..EngineConfig::default()
3827        };
3828
3829        let id = store
3830            .upsert_external(&crate::store::UpsertExternal {
3831                repo: "owner/repo",
3832                ext_id: "103",
3833                title: "Hung review session",
3834                body: "",
3835                author: "",
3836                url: "",
3837                labels: &[],
3838                origin: "github",
3839            })
3840            .await
3841            .unwrap();
3842        store
3843            .update_status(id, crate::store::TaskStatus::InReview)
3844            .await
3845            .unwrap();
3846        set_in_review_entered_at_past(&store, id).await;
3847        store
3848            .set_fields(id, &[("pr_number", serde_json::json!(43))])
3849            .await
3850            .unwrap();
3851
3852        let session_name = tmux.session_name("owner/repo", "103-review");
3853        let mut session_map = std::collections::HashMap::new();
3854        session_map.insert(session_name, true);
3855
3856        tick_recover_stuck_tasks(
3857            &backend,
3858            &tmux,
3859            "owner/repo",
3860            &task_manager,
3861            &config,
3862            &store,
3863            &session_map,
3864            &Arc::new(DashMap::new()),
3865        )
3866        .await
3867        .unwrap();
3868
3869        let updates = status_updates.lock().unwrap();
3870        assert_eq!(
3871            updates.len(),
3872            1,
3873            "a live hung review session must be reclaimed once the in_review entry ages past stuck_timeout"
3874        );
3875        assert_eq!(updates[0].0, "103");
3876        assert_eq!(updates[0].1, Status::NeedsReview);
3877    }
3878
3879    #[tokio::test]
3880    async fn untracked_repo_recovery_resets_stale_in_progress_to_new() {
3881        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3882        let tmux = Arc::new(TmuxManager::new());
3883        let config = EngineConfig {
3884            no_session_stuck_timeout: 600,
3885            stuck_timeout: 1800,
3886            ..EngineConfig::default()
3887        };
3888
3889        // Cross-repo task should still be recovered even when current tick repo is owner/repo.
3890        let id = store
3891            .upsert_external(&crate::store::UpsertExternal {
3892                repo: "owner/other",
3893                ext_id: "777",
3894                title: "Cross-repo stuck task",
3895                body: "",
3896                author: "",
3897                url: "",
3898                labels: &[],
3899                origin: "github",
3900            })
3901            .await
3902            .unwrap();
3903        store
3904            .update_status(id, crate::store::TaskStatus::InProgress)
3905            .await
3906            .unwrap();
3907        set_task_updated_at_past(&store, id).await;
3908
3909        let active = std::collections::HashSet::from(["owner/repo".to_string()]);
3910        recover_untracked_repo_stuck_tasks(
3911            &tmux,
3912            &config,
3913            &store,
3914            &std::collections::HashMap::new(),
3915            &Arc::new(DashMap::new()),
3916            &active,
3917        )
3918        .await;
3919
3920        let task = store.get(id).await.unwrap();
3921        assert_eq!(
3922            task.status,
3923            crate::store::TaskStatus::New,
3924            "cross-repo stale in_progress task should be reset to New"
3925        );
3926        assert_eq!(task.route_attempts, 0, "route attempts should be cleared");
3927        assert!(task.agent.is_none(), "agent should be cleared");
3928        assert!(task.model.is_none(), "model should be cleared");
3929    }
3930
3931    #[tokio::test]
3932    async fn untracked_repo_recovery_leaves_active_repos_to_their_own_pass() {
3933        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3934        let tmux = Arc::new(TmuxManager::new());
3935        let config = EngineConfig {
3936            no_session_stuck_timeout: 600,
3937            stuck_timeout: 1800,
3938            ..EngineConfig::default()
3939        };
3940        let id = store
3941            .upsert_external(&crate::store::UpsertExternal {
3942                repo: "owner/other",
3943                ext_id: "778",
3944                title: "Stuck task in an active repo",
3945                body: "",
3946                author: "",
3947                url: "",
3948                labels: &[],
3949                origin: "github",
3950            })
3951            .await
3952            .unwrap();
3953        store
3954            .update_status(id, crate::store::TaskStatus::InProgress)
3955            .await
3956            .unwrap();
3957        set_task_updated_at_past(&store, id).await;
3958
3959        // Per-project passes run concurrently, so only the task's own pass may reclaim it
3960        let active =
3961            std::collections::HashSet::from(["owner/repo".to_string(), "owner/other".to_string()]);
3962        recover_untracked_repo_stuck_tasks(
3963            &tmux,
3964            &config,
3965            &store,
3966            &std::collections::HashMap::new(),
3967            &Arc::new(DashMap::new()),
3968            &active,
3969        )
3970        .await;
3971
3972        assert_eq!(
3973            store.get(id).await.unwrap().status,
3974            crate::store::TaskStatus::InProgress
3975        );
3976    }
3977
3978    #[tokio::test]
3979    async fn recover_stuck_tasks_finalizes_open_task_runs_rows() {
3980        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3981        let mock = MockBackend::new();
3982        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
3983        let task_manager = Arc::new(TaskManager::with_store(
3984            std::sync::Arc::clone(&backend),
3985            std::sync::Arc::clone(&store),
3986            "owner/repo".to_string(),
3987        ));
3988        let tmux = Arc::new(TmuxManager::new());
3989        let config = EngineConfig {
3990            no_session_stuck_timeout: 600,
3991            stuck_timeout: 1800,
3992            ..EngineConfig::default()
3993        };
3994
3995        let id = store
3996            .create_internal("owner/repo", "Stuck with open run", "", "cron", "1", None)
3997            .await
3998            .unwrap();
3999        store
4000            .set_fields(
4001                id,
4002                &[
4003                    ("agent", serde_json::json!("claude")),
4004                    ("model", serde_json::json!("sonnet")),
4005                    ("route_attempts", serde_json::json!(1)),
4006                ],
4007            )
4008            .await
4009            .unwrap();
4010        store
4011            .update_status(id, crate::store::TaskStatus::InProgress)
4012            .await
4013            .unwrap();
4014        set_task_updated_at_past(&store, id).await;
4015
4016        // Start an agent run that will be orphaned by recovery
4017        store
4018            .start_run(&crate::store::StartRun {
4019                task_id: id,
4020                attempt: 1,
4021                run_type: "agent",
4022                agent: "claude",
4023                model: "sonnet",
4024                command: "claude --model sonnet",
4025                prompt: "fix it",
4026            })
4027            .await
4028            .unwrap();
4029
4030        tick_recover_stuck_tasks(
4031            &backend,
4032            &tmux,
4033            "owner/repo",
4034            &task_manager,
4035            &config,
4036            &store,
4037            &std::collections::HashMap::new(),
4038            &Arc::new(DashMap::new()),
4039        )
4040        .await
4041        .unwrap();
4042
4043        let task = store.get(id).await.unwrap();
4044        assert_eq!(task.status, crate::store::TaskStatus::New);
4045
4046        let runs = store.get_runs(id).await.unwrap();
4047        assert_eq!(runs.len(), 1);
4048        assert_eq!(runs[0].outcome, "aborted");
4049        assert!(runs[0].completed_at.is_some());
4050        assert_eq!(
4051            runs[0].error,
4052            "stuck-task recovery: internal no session found"
4053        );
4054    }
4055
4056    /// Regression test for #3518: a completed agent run must not be discarded by the
4057    /// stuck-task reclaim just because its tmux session has already exited.
4058    ///
4059    /// In production, sessions are created without `remain-on-exit`, so once the agent
4060    /// process exits tmux removes the session immediately — there is no observable
4061    /// "dead pane" state for `tick_check_session_completions` to catch, and the runner's
4062    /// own `store_touch_updated_at` only fires after its own poll notices the exit (up to
4063    /// its 5s poll interval) plus post-processing. For a task whose total runtime already
4064    /// exceeds `no_session_stuck_timeout`, the very next tick that observes the (now
4065    /// absent) session would otherwise reclaim it instantly — discarding a successful run.
4066    ///
4067    /// The fix: `tick_recover_stuck_tasks` skips the no-session reclaim while the task's
4068    /// dispatch guard (the in-memory `dispatching` map keyed by "{repo}/{task_id}",
4069    /// populated for the entire duration of `runner.run_with_context`) is still held —
4070    /// i.e. while this process is still actively running the task, regardless of tmux
4071    /// session state.
4072    #[tokio::test]
4073    async fn recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held() {
4074        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4075        let mock = MockBackend::new();
4076        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4077        let task_manager = Arc::new(TaskManager::with_store(
4078            std::sync::Arc::clone(&backend),
4079            std::sync::Arc::clone(&store),
4080            "owner/repo".to_string(),
4081        ));
4082        let tmux = Arc::new(TmuxManager::new());
4083        let config = EngineConfig {
4084            no_session_stuck_timeout: 600,
4085            stuck_timeout: 1800,
4086            ..EngineConfig::default()
4087        };
4088
4089        let id = store
4090            .create_internal(
4091                "owner/repo",
4092                "Long-running daily job",
4093                "",
4094                "cron",
4095                "1",
4096                None,
4097            )
4098            .await
4099            .unwrap();
4100        store
4101            .set_fields(
4102                id,
4103                &[
4104                    ("agent", serde_json::json!("claude")),
4105                    ("model", serde_json::json!("sonnet")),
4106                    ("route_attempts", serde_json::json!(1)),
4107                ],
4108            )
4109            .await
4110            .unwrap();
4111        store
4112            .update_status(id, crate::store::TaskStatus::InProgress)
4113            .await
4114            .unwrap();
4115        // Simulate a task that has already run well past no_session_stuck_timeout,
4116        // as a legitimately long agent run would.
4117        set_task_updated_at_past(&store, id).await;
4118
4119        // Simulate the runner still actively processing this task: its dispatch guard
4120        // is held even though the tmux session (empty session_map below) is already gone.
4121        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4122        dispatching.insert(
4123            format!("owner/repo/internal:{id}"),
4124            format!("internal:{id}"),
4125        );
4126
4127        tick_recover_stuck_tasks(
4128            &backend,
4129            &tmux,
4130            "owner/repo",
4131            &task_manager,
4132            &config,
4133            &store,
4134            &std::collections::HashMap::new(),
4135            &dispatching,
4136        )
4137        .await
4138        .unwrap();
4139
4140        let task = store.get(id).await.unwrap();
4141        assert_eq!(
4142            task.status,
4143            crate::store::TaskStatus::InProgress,
4144            "task should not be reclaimed while its dispatch guard is still held"
4145        );
4146
4147        // No run should have been finalized/aborted either.
4148        let runs = store.get_runs(id).await.unwrap();
4149        assert!(runs.is_empty());
4150
4151        // Once the guard is released (runner finished), the very same conditions should
4152        // reclaim the task as before — the fix must not disable the safety net entirely.
4153        dispatching.clear();
4154        tick_recover_stuck_tasks(
4155            &backend,
4156            &tmux,
4157            "owner/repo",
4158            &task_manager,
4159            &config,
4160            &store,
4161            &std::collections::HashMap::new(),
4162            &dispatching,
4163        )
4164        .await
4165        .unwrap();
4166        let task = store.get(id).await.unwrap();
4167        assert_eq!(
4168            task.status,
4169            crate::store::TaskStatus::New,
4170            "task should still be reclaimed once the dispatch guard is released"
4171        );
4172    }
4173
4174    /// Concurrency regression test for issue #3526: the reclaim race recurred twice
4175    /// (internal:156563 in #3523, internal:156673 in #3526) *after* the #3519/#3525
4176    /// fix landed, in production under the real multi-threaded tokio runtime. The
4177    /// existing `recover_stuck_tasks_skips_reclaim_while_dispatch_guard_held` test
4178    /// above only exercises sequential `.await` calls on a single-threaded test
4179    /// runtime (`#[tokio::test]` defaults to `flavor = "current_thread"`), which can
4180    /// never observe genuine cross-thread races in the shared `dispatching` DashMap.
4181    ///
4182    /// This test claims the guard exactly the way `dispatch_tasks_for_repo` does
4183    /// (atomic `DashMap::entry` claim, `DispatchGuard` RAII release), hands it to a
4184    /// spawned task running on a *different* worker thread that holds it until
4185    /// explicitly told to release, and hammers the real `tick_recover_stuck_tasks`
4186    /// from the main test task hundreds of times while the guard is deterministically
4187    /// still held. If there were a visibility gap between `DashMap::entry().insert()`
4188    /// on one thread and `DashMap::contains_key()` on another, this test would catch it.
4189    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
4190    async fn recover_stuck_tasks_never_reclaims_while_guard_held_under_real_concurrency() {
4191        const HAMMER_ITERATIONS: u32 = 300;
4192
4193        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4194        let mock = MockBackend::new();
4195        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4196        let task_manager = Arc::new(TaskManager::with_store(
4197            std::sync::Arc::clone(&backend),
4198            std::sync::Arc::clone(&store),
4199            "owner/repo".to_string(),
4200        ));
4201        let tmux = Arc::new(TmuxManager::new());
4202        let config = EngineConfig {
4203            no_session_stuck_timeout: 600,
4204            stuck_timeout: 1800,
4205            ..EngineConfig::default()
4206        };
4207
4208        let id = store
4209            .create_internal(
4210                "owner/repo",
4211                "Long-running daily job",
4212                "",
4213                "cron",
4214                "1",
4215                None,
4216            )
4217            .await
4218            .unwrap();
4219        store
4220            .set_fields(
4221                id,
4222                &[
4223                    ("agent", serde_json::json!("claude")),
4224                    ("model", serde_json::json!("sonnet")),
4225                    ("route_attempts", serde_json::json!(1)),
4226                ],
4227            )
4228            .await
4229            .unwrap();
4230        store
4231            .update_status(id, crate::store::TaskStatus::InProgress)
4232            .await
4233            .unwrap();
4234        set_task_updated_at_past(&store, id).await;
4235
4236        let dispatch_key = format!("owner/repo/internal:{id}");
4237        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
4238
4239        // Claim the guard synchronously, on this task, exactly like
4240        // dispatch_tasks_for_repo does: the atomic entry-claim always happens before
4241        // tokio::spawn hands the guard to the background task. (Claiming inside the
4242        // spawned task itself would race the hammer loop below against tokio's own
4243        // scheduling of that task onto a worker thread — a test artifact, not the
4244        // real production ordering, which guarantees claim-before-spawn.)
4245        let claim_task_id = format!("internal:{id}");
4246        {
4247            use dashmap::mapref::entry::Entry;
4248            match dispatching.entry(dispatch_key.clone()) {
4249                Entry::Vacant(slot) => {
4250                    slot.insert(claim_task_id);
4251                }
4252                Entry::Occupied(_) => panic!("dispatch key already claimed"),
4253            }
4254        }
4255        let guard = DispatchGuard::new(Arc::clone(&dispatching), dispatch_key.clone());
4256
4257        // Hold the guard on a spawned task — a different worker thread under the
4258        // multi-thread runtime — mirroring the real runner holding it across genuine
4259        // async work (worktree setup, agent session, response handling) inside
4260        // `tokio::spawn(... run_with_context ...)`. It only drops the guard once
4261        // explicitly told to via `release_rx`, so its lifetime is fully controlled
4262        // by this test rather than raced against a timer — the release-signal
4263        // ordering was itself a source of false positives in an earlier version of
4264        // this test (a `sleep`-then-drop holder can legitimately release the guard
4265        // in the gap between the hammer loop's last check and its next iteration,
4266        // which is a correct outcome, not a bug, but made the assertion flaky).
4267        let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
4268        let holder = tokio::spawn(async move {
4269            let _ = release_rx.await;
4270            drop(guard);
4271        });
4272
4273        // Hammer the real reclaim path concurrently from the main task while the
4274        // guard is deterministically still held (the holder task is parked on
4275        // `release_rx` and cannot drop the guard until we signal it below). Each
4276        // iteration's `tick_recover_stuck_tasks` call may itself run on any worker
4277        // thread the tokio scheduler picks, independent of the thread that claimed
4278        // the guard — this is what actually exercises cross-thread DashMap
4279        // visibility, not wall-clock racing against a timer.
4280        for i in 0..HAMMER_ITERATIONS {
4281            tick_recover_stuck_tasks(
4282                &backend,
4283                &tmux,
4284                "owner/repo",
4285                &task_manager,
4286                &config,
4287                &store,
4288                &std::collections::HashMap::new(),
4289                &dispatching,
4290            )
4291            .await
4292            .unwrap();
4293
4294            let task = store.get(id).await.unwrap();
4295            assert_eq!(
4296                task.status,
4297                crate::store::TaskStatus::InProgress,
4298                "task must never be reclaimed while the dispatch guard is deterministically \
4299                 still held, even under real cross-thread concurrency (dispatch_key={dispatch_key}, \
4300                 iteration={i}, contains_key_now={}, map_len_now={})",
4301                dispatching.contains_key(&dispatch_key),
4302                dispatching.len(),
4303            );
4304            tokio::task::yield_now().await;
4305        }
4306
4307        // Now release the guard and confirm the holder task actually completed.
4308        let _ = release_tx.send(());
4309        holder.await.unwrap();
4310
4311        // Guard released — the safety net must still reclaim as before.
4312        tick_recover_stuck_tasks(
4313            &backend,
4314            &tmux,
4315            "owner/repo",
4316            &task_manager,
4317            &config,
4318            &store,
4319            &std::collections::HashMap::new(),
4320            &dispatching,
4321        )
4322        .await
4323        .unwrap();
4324        let task = store.get(id).await.unwrap();
4325        assert_eq!(
4326            task.status,
4327            crate::store::TaskStatus::New,
4328            "task should be reclaimed once the dispatch guard is released"
4329        );
4330    }
4331
4332    #[tokio::test]
4333    async fn recover_stuck_tasks_skips_external_in_review_waiting_for_human_review() {
4334        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4335        let mock = MockBackend::new();
4336        let status_updates = std::sync::Arc::clone(&mock.status_updates);
4337        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4338        let task_manager = Arc::new(TaskManager::with_store(
4339            std::sync::Arc::clone(&backend),
4340            std::sync::Arc::clone(&store),
4341            "owner/repo".to_string(),
4342        ));
4343        let tmux = Arc::new(TmuxManager::new());
4344        let config = EngineConfig {
4345            no_session_stuck_timeout: 600,
4346            stuck_timeout: 1800,
4347            ..EngineConfig::default()
4348        };
4349
4350        let id = store
4351            .upsert_external(&crate::store::UpsertExternal {
4352                repo: "owner/repo",
4353                ext_id: "101",
4354                title: "Waiting For Human Review",
4355                body: "",
4356                author: "",
4357                url: "",
4358                labels: &[],
4359                origin: "github",
4360            })
4361            .await
4362            .unwrap();
4363        store
4364            .update_status(id, crate::store::TaskStatus::InReview)
4365            .await
4366            .unwrap();
4367        set_task_updated_at_past(&store, id).await;
4368
4369        tick_recover_stuck_tasks(
4370            &backend,
4371            &tmux,
4372            "owner/repo",
4373            &task_manager,
4374            &config,
4375            &store,
4376            &std::collections::HashMap::new(),
4377            &Arc::new(DashMap::new()),
4378        )
4379        .await
4380        .unwrap();
4381
4382        let updates = status_updates.lock().unwrap();
4383        assert!(
4384            updates.is_empty(),
4385            "InReview task waiting for human review should not be reset"
4386        );
4387    }
4388
4389    #[tokio::test]
4390    async fn unblock_parents_no_blocked_tasks() {
4391        let mock = MockBackend::new(); // no blocked tasks
4392        let status_updates = std::sync::Arc::clone(&mock.status_updates);
4393        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4394        let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
4395        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4396
4397        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4398            .await
4399            .unwrap();
4400
4401        assert!(status_updates.lock().unwrap().is_empty());
4402    }
4403
4404    #[tokio::test]
4405    async fn unblock_parents_handles_multiple_blocked_tasks() {
4406        let mut mock = MockBackend::new();
4407
4408        // Task 40: all children done → should unblock
4409        let t40 = make_task("40", &["status:blocked"]);
4410        mock.blocked_tasks.push(t40);
4411        mock.sub_issues
4412            .insert("40".to_string(), vec![ExternalId("41".to_string())]);
4413        mock.tasks_by_id
4414            .insert("41".to_string(), make_task("41", &["status:done"]));
4415
4416        // Task 50: child not done → should stay blocked
4417        let t50 = make_task("50", &["status:blocked"]);
4418        mock.blocked_tasks.push(t50);
4419        mock.sub_issues
4420            .insert("50".to_string(), vec![ExternalId("51".to_string())]);
4421        mock.tasks_by_id
4422            .insert("51".to_string(), make_task("51", &["status:in_progress"]));
4423
4424        let status_updates = std::sync::Arc::clone(&mock.status_updates);
4425        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4426        let task_manager = make_task_manager(std::sync::Arc::clone(&backend));
4427        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4428
4429        tick_unblock_parents(&backend, &task_manager, &store, "test/repo")
4430            .await
4431            .unwrap();
4432
4433        let updates = status_updates.lock().unwrap();
4434        assert_eq!(updates.len(), 1, "only one parent should be unblocked");
4435        assert_eq!(updates[0].0, "40", "task 40 should be unblocked");
4436        assert_eq!(updates[0].1, Status::New);
4437    }
4438
4439    /// `tick_all` reads leadership and the global sweeps from the first project's store, so
4440    /// an empty project list must return before any of that, and touch nothing.
4441    #[tokio::test]
4442    async fn tick_all_is_a_no_op_without_projects() {
4443        let transport = Arc::new(Transport::new());
4444        let capture = Arc::new(CaptureService::new(std::sync::Arc::clone(&transport)));
4445        let tmux = Arc::new(TmuxManager::new());
4446        let semaphore = Arc::new(Semaphore::new(1));
4447        let router = Arc::new(tokio::sync::RwLock::new(cooled_review_router_for_test()));
4448        let (weight_tx, mut weight_rx) = mpsc::channel(8);
4449        let dispatching = Arc::new(DashMap::new());
4450        let auto_merge_in_flight = Arc::new(DashSet::new());
4451
4452        tick_all(
4453            &[],
4454            &tmux,
4455            &capture,
4456            &semaphore,
4457            &EngineConfig::default(),
4458            &router,
4459            &weight_tx,
4460            &dispatching,
4461            Some(&transport),
4462            &std::collections::HashSet::new(),
4463            &auto_merge_in_flight,
4464        )
4465        .await;
4466
4467        assert!(dispatching.is_empty(), "no project, so no dispatch claimed");
4468        assert!(
4469            weight_rx.try_recv().is_err(),
4470            "no project, so no weight signal"
4471        );
4472    }
4473
4474    /// The store-wide sweeps must run once per tick, not once per project. The NeedsReview
4475    /// refire sweep reads `needs_review_refires` and then increments it, so when it ran
4476    /// inside the per-project join every project saw the same count and spent one of the
4477    /// five refire attempts — four projects burned four attempts in a single tick (#3864).
4478    #[serial_test::serial(cooldown_state)]
4479    #[tokio::test]
4480    async fn tick_all_runs_the_global_needs_review_sweep_once_for_two_projects() {
4481        use crate::engine::router::{Router, RouterConfig};
4482        use crate::store::UpsertExternal;
4483
4484        crate::engine::cooldown::reset_global_state().await;
4485
4486        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4487        let id = store
4488            .upsert_external(&UpsertExternal {
4489                repo: "owner/one",
4490                ext_id: "7",
4491                title: "Stale needs_review task",
4492                body: "",
4493                author: "user",
4494                url: "https://github.com/owner/one/issues/7",
4495                labels: &[],
4496                origin: "github",
4497            })
4498            .await
4499            .unwrap();
4500        store
4501            .update_status(id, crate::store::TaskStatus::NeedsReview)
4502            .await
4503            .unwrap();
4504        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
4505            .bind(id)
4506            .execute(store.pool())
4507            .await
4508            .unwrap();
4509
4510        let engines: Vec<ProjectEngine> = ["owner/one", "owner/two"]
4511            .iter()
4512            .map(|repo| {
4513                let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4514                ProjectEngine {
4515                    repo: repo.to_string(),
4516                    project_dir: std::path::PathBuf::from("/nonexistent"),
4517                    backend: std::sync::Arc::clone(&backend),
4518                    task_manager: Arc::new(TaskManager::with_store(
4519                        backend,
4520                        std::sync::Arc::clone(&store),
4521                        repo.to_string(),
4522                    )),
4523                    runner: Arc::new(crate::engine::runner::TaskRunner::new(repo.to_string())),
4524                    store: std::sync::Arc::clone(&store),
4525                }
4526            })
4527            .collect();
4528
4529        // A healthy review agent, so the sweep does not skip the pass
4530        let mut config = RouterConfig::default();
4531        config
4532            .model_map
4533            .entry("review".to_string())
4534            .or_default()
4535            .insert("claude".to_string(), vec!["sonnet".to_string()]);
4536        let router = Arc::new(tokio::sync::RwLock::new(Router::new_for_test(
4537            config,
4538            vec!["claude".to_string()],
4539        )));
4540
4541        let transport = Arc::new(Transport::new());
4542        let capture = Arc::new(CaptureService::new(std::sync::Arc::clone(&transport)));
4543        let (weight_tx, _weight_rx) = mpsc::channel(8);
4544
4545        tick_all(
4546            &engines,
4547            &Arc::new(TmuxManager::new()),
4548            &capture,
4549            &Arc::new(Semaphore::new(1)),
4550            &EngineConfig::default(),
4551            &router,
4552            &weight_tx,
4553            &Arc::new(DashMap::new()),
4554            Some(&transport),
4555            &engines.iter().map(|e| e.repo.clone()).collect(),
4556            &Arc::new(DashSet::new()),
4557        )
4558        .await;
4559
4560        assert_eq!(
4561            store.get(id).await.unwrap().needs_review_refires,
4562            1,
4563            "two projects must share one pass of the global NeedsReview sweep"
4564        );
4565    }
4566
4567    /// `tick_all` must not reap tmux sessions it did not start. Startup cleanup used to
4568    /// live here, and nextest's per-test process made the once-flag kill live agents (#3889)
4569    #[tokio::test]
4570    async fn tick_all_does_not_kill_sessions_it_did_not_create() {
4571        let tmux = Arc::new(TmuxManager::new());
4572        let Ok(session_name) = tmux
4573            .create_session("other/foreign", "3889", "/tmp", "sleep 30", &[])
4574            .await
4575        else {
4576            eprintln!("Skipping test: tmux not available or failed to create test session");
4577            return;
4578        };
4579        let _guard = SessionGuard(session_name.clone());
4580
4581        // Cross a unix-second boundary so a reintroduced startup_cleanup would treat
4582        // this session as stale (created_at < now)
4583        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
4584
4585        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4586        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
4587        let engines = vec![ProjectEngine {
4588            repo: "owner/one".to_string(),
4589            project_dir: std::path::PathBuf::from("/nonexistent"),
4590            backend: std::sync::Arc::clone(&backend),
4591            task_manager: Arc::new(TaskManager::with_store(
4592                backend,
4593                std::sync::Arc::clone(&store),
4594                "owner/one".to_string(),
4595            )),
4596            runner: Arc::new(crate::engine::runner::TaskRunner::new(
4597                "owner/one".to_string(),
4598            )),
4599            store,
4600        }];
4601
4602        let transport = Arc::new(Transport::new());
4603        let capture = Arc::new(CaptureService::new(std::sync::Arc::clone(&transport)));
4604        let (weight_tx, _weight_rx) = mpsc::channel(8);
4605        let router = Arc::new(tokio::sync::RwLock::new(cooled_review_router_for_test()));
4606
4607        tick_all(
4608            &engines,
4609            &tmux,
4610            &capture,
4611            &Arc::new(Semaphore::new(1)),
4612            &EngineConfig::default(),
4613            &router,
4614            &weight_tx,
4615            &Arc::new(DashMap::new()),
4616            Some(&transport),
4617            &engines.iter().map(|e| e.repo.clone()).collect(),
4618            &Arc::new(DashSet::new()),
4619        )
4620        .await;
4621
4622        assert!(
4623            tmux.session_exists(&session_name).await,
4624            "tick_all must not kill a tmux session it did not start"
4625        );
4626    }
4627
4628    #[tokio::test]
4629    async fn detect_silent_agents_skips_backend_for_internal_tasks() {
4630        let transport = Arc::new(Transport::new());
4631        let capture = Arc::new(CaptureService::new(transport));
4632        let mock = Arc::new(MockBackend::new());
4633        let backend: Arc<dyn ExternalBackend> = Arc::<MockBackend>::clone(&mock);
4634        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4635        let task_manager = Arc::new(TaskManager::with_store(
4636            std::sync::Arc::clone(&backend),
4637            std::sync::Arc::clone(&store),
4638            "owner/repo".to_string(),
4639        ));
4640        let tmux = Arc::new(TmuxManager::new());
4641        let config = EngineConfig {
4642            silence_grace_period: 0,
4643            ..EngineConfig::default()
4644        };
4645
4646        let internal_id = store
4647            .create_internal("owner/repo", "Silent internal", "", "cron", "1", None)
4648            .await
4649            .unwrap();
4650        store
4651            .update_status(internal_id, crate::store::TaskStatus::InProgress)
4652            .await
4653            .unwrap();
4654
4655        let task_id = format!("internal:{internal_id}");
4656        capture
4657            .register_session("owner/repo", &task_id, "orch-test-internal")
4658            .await;
4659        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
4660
4661        tick_detect_silent_agents(
4662            &tmux,
4663            "owner/repo",
4664            &capture,
4665            &backend,
4666            &task_manager,
4667            &config,
4668            &store,
4669        )
4670        .await
4671        .unwrap();
4672
4673        assert!(
4674            mock.removed_labels.lock().unwrap().is_empty(),
4675            "internal tasks should skip backend label removal"
4676        );
4677        assert!(
4678            mock.posted_comments.lock().unwrap().is_empty(),
4679            "internal tasks should skip backend comments"
4680        );
4681
4682        let task = store.get(internal_id).await.unwrap();
4683        // Silence detection now uses failover: if a fallback agent is found → Routed,
4684        // otherwise → New (reset for re-routing when cooldowns expire). Either is
4685        // acceptable; the key invariant is that the task is no longer InProgress.
4686        assert!(
4687            matches!(
4688                task.status,
4689                crate::store::TaskStatus::Routed | crate::store::TaskStatus::New
4690            ),
4691            "internal task should be routed to fallback or reset to new, got {:?}",
4692            task.status
4693        );
4694    }
4695
4696    #[tokio::test]
4697    async fn detect_silent_agents_finalizes_open_task_runs_rows() {
4698        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4699        let mock = MockBackend::new();
4700        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4701        let task_manager = Arc::new(TaskManager::with_store(
4702            std::sync::Arc::clone(&backend),
4703            std::sync::Arc::clone(&store),
4704            "owner/repo".to_string(),
4705        ));
4706        let transport = Arc::new(Transport::new());
4707        let capture = Arc::new(CaptureService::new(transport));
4708        let tmux = Arc::new(TmuxManager::new());
4709        let config = EngineConfig {
4710            silence_grace_period: 60,
4711            silence_cooldown: 300,
4712            ..EngineConfig::default()
4713        };
4714
4715        let internal_id = store
4716            .create_internal("owner/repo", "Silent with open run", "", "cron", "1", None)
4717            .await
4718            .unwrap();
4719        store
4720            .set_fields(
4721                internal_id,
4722                &[
4723                    ("agent", serde_json::json!("claude")),
4724                    ("model", serde_json::json!("sonnet")),
4725                ],
4726            )
4727            .await
4728            .unwrap();
4729        store
4730            .update_status(internal_id, crate::store::TaskStatus::InProgress)
4731            .await
4732            .unwrap();
4733
4734        let task_id = format!("internal:{internal_id}");
4735        capture
4736            .register_session("owner/repo", &task_id, "orch-test-silent-run")
4737            .await;
4738        capture
4739            .set_buffer_state_for_test(
4740                "owner/repo",
4741                &task_id,
4742                false,
4743                false,
4744                chrono::Utc::now() - chrono::Duration::seconds(120),
4745            )
4746            .await;
4747
4748        // Start an agent run that will be orphaned by silence detection
4749        store
4750            .start_run(&crate::store::StartRun {
4751                task_id: internal_id,
4752                attempt: 1,
4753                run_type: "agent",
4754                agent: "claude",
4755                model: "sonnet",
4756                command: "claude --model sonnet",
4757                prompt: "fix it",
4758            })
4759            .await
4760            .unwrap();
4761
4762        tick_detect_silent_agents(
4763            &tmux,
4764            "owner/repo",
4765            &capture,
4766            &backend,
4767            &task_manager,
4768            &config,
4769            &store,
4770        )
4771        .await
4772        .unwrap();
4773
4774        let runs = store.get_runs(internal_id).await.unwrap();
4775        assert_eq!(runs.len(), 1);
4776        assert_eq!(runs[0].outcome, "aborted");
4777        assert!(runs[0].completed_at.is_some());
4778        assert_eq!(runs[0].error, "silence detection: session killed");
4779    }
4780
4781    /// Regression test for the race between Phase 1 (tick_check_session_completions)
4782    /// and Phase 2 (tick_recover_stuck_tasks).
4783    ///
4784    /// Before the fix, Phase 1 killed the tmux session without touching updated_at.
4785    /// Phase 2 could then see status=in_progress, has_session=false, and an old
4786    /// updated_at, and would reset the task to New — even though the runner's
4787    /// wait_for_completion() poll had not yet returned.
4788    ///
4789    /// The fix: Phase 1 touches updated_at as soon as it detects the session is done.
4790    /// This test verifies that after the touch, Phase 2 does NOT reclaim the task.
4791    #[tokio::test]
4792    async fn phase1_session_completion_touch_prevents_phase2_reclaim() {
4793        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4794        let mock = MockBackend::new();
4795        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4796        let task_manager = Arc::new(TaskManager::with_store(
4797            std::sync::Arc::clone(&backend),
4798            std::sync::Arc::clone(&store),
4799            "owner/repo".to_string(),
4800        ));
4801        let tmux = Arc::new(TmuxManager::new());
4802        let config = EngineConfig {
4803            no_session_stuck_timeout: 600,
4804            stuck_timeout: 1800,
4805            ..EngineConfig::default()
4806        };
4807
4808        // Create an in-progress internal task with old updated_at.
4809        // Without the fix this task would be reclaimed by Phase 2 (age >> threshold,
4810        // no session running in the test environment).
4811        let id = store
4812            .create_internal("owner/repo", "Completed task", "", "cron", "1", None)
4813            .await
4814            .unwrap();
4815        store
4816            .update_status(id, crate::store::TaskStatus::InProgress)
4817            .await
4818            .unwrap();
4819        set_task_updated_at_past(&store, id).await;
4820
4821        // Simulate Phase 1: touch updated_at when the session is detected as done.
4822        // In production this happens inside tick_check_session_completions.
4823        let task_id = format!("internal:{id}");
4824        store_touch_updated_at(&Some(Arc::clone(&store)), "owner/repo", &task_id).await;
4825
4826        // Phase 2 runs — updated_at is fresh so the task must NOT be reclaimed.
4827        tick_recover_stuck_tasks(
4828            &backend,
4829            &tmux,
4830            "owner/repo",
4831            &task_manager,
4832            &config,
4833            &store,
4834            &std::collections::HashMap::new(),
4835            &Arc::new(DashMap::new()),
4836        )
4837        .await
4838        .unwrap();
4839
4840        let task = store.get(id).await.unwrap();
4841        assert_eq!(
4842            task.status,
4843            crate::store::TaskStatus::InProgress,
4844            "Phase 2 must not reclaim a task whose updated_at was just touched by Phase 1"
4845        );
4846    }
4847
4848    #[tokio::test]
4849    async fn stuck_task_timing_treats_dead_session_as_missing() {
4850        let tmux = Arc::new(TmuxManager::new());
4851        let repo = "owner/repo";
4852        let task_id = "internal:33498";
4853        let session_name = tmux.session_name(repo, task_id);
4854
4855        let create_result = tmux
4856            .command()
4857            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
4858            .output()
4859            .await;
4860
4861        match create_result {
4862            Ok(o) if o.status.success() => {}
4863            _ => {
4864                eprintln!("Skipping test: tmux not available or failed to create test session");
4865                return;
4866            }
4867        }
4868
4869        let _guard = SessionGuard(session_name.clone());
4870
4871        let set_option_result = tmux
4872            .command()
4873            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
4874            .output()
4875            .await;
4876        if !matches!(set_option_result, Ok(ref o) if o.status.success()) {
4877            eprintln!("Skipping test: unable to set tmux remain-on-exit option");
4878            return;
4879        }
4880
4881        let send_exit_result = tmux
4882            .command()
4883            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
4884            .output()
4885            .await;
4886        if !matches!(send_exit_result, Ok(ref o) if o.status.success()) {
4887            eprintln!("Skipping test: unable to exit tmux pane");
4888            return;
4889        }
4890
4891        // Poll until pane is dead (up to 2s) — fixed sleep is unreliable under load.
4892        for _ in 0..20 {
4893            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4894            if !tmux.session_is_running(&session_name).await {
4895                break;
4896            }
4897        }
4898
4899        // Also wait for batch_session_active to reflect the dead state.
4900        // session_is_running uses `list-panes -t <session>` which may settle
4901        // before the global `list-panes -a` snapshot used by the production code.
4902        // On CI under load these two tmux commands can transiently disagree.
4903        let mut session_map = tmux.batch_session_active().await;
4904        for _ in 0..20 {
4905            if !session_map.get(&session_name).copied().unwrap_or(false) {
4906                break;
4907            }
4908            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
4909            session_map = tmux.batch_session_active().await;
4910        }
4911        if session_map.get(&session_name).copied().unwrap_or(false) {
4912            eprintln!(
4913                "Skipping test: session still alive in batch_session_active after retries (CI tmux lag)"
4914            );
4915            return;
4916        }
4917
4918        let config = EngineConfig {
4919            no_session_stuck_timeout: 600,
4920            stuck_timeout: 1800,
4921            ..EngineConfig::default()
4922        };
4923        let updated_at = (chrono::Utc::now()
4924            - chrono::Duration::seconds(
4925                i64::try_from(config.no_session_stuck_timeout).expect("timeout fits in i64") + 5,
4926            ))
4927        .to_rfc3339();
4928
4929        let timing = stuck_task_timing_from_map(
4930            &tmux,
4931            repo,
4932            task_id,
4933            &updated_at,
4934            &config,
4935            "parse failure",
4936            &session_map,
4937        )
4938        .expect("dead session should be treated as missing");
4939
4940        assert!(!timing.has_session);
4941        assert_eq!(timing.threshold, config.no_session_stuck_timeout);
4942    }
4943
4944    #[tokio::test]
4945    #[serial_test::serial(suspend_state)]
4946    async fn stuck_task_timing_discounts_host_suspend_gap() {
4947        super::super::suspend::clear_for_test();
4948
4949        let tmux = Arc::new(TmuxManager::new());
4950        let repo = "owner/repo";
4951        let task_id = "internal:99001";
4952        let session_map = std::collections::HashMap::new();
4953
4954        let config = EngineConfig {
4955            no_session_stuck_timeout: 1800,
4956            stuck_timeout: 1800,
4957            ..EngineConfig::default()
4958        };
4959
4960        // Task was last touched 33 minutes ago (age_mins=33 matches the reported
4961        // false-positive), but the host was suspended for 30 of those minutes —
4962        // real elapsed runtime is only 3 minutes, well under the 30-minute threshold.
4963        let updated_at = (chrono::Utc::now() - chrono::Duration::minutes(33)).to_rfc3339();
4964        let suspend_gap_detected_at = chrono::Utc::now() - chrono::Duration::minutes(1);
4965        super::super::suspend::inject_gap_for_test(
4966            suspend_gap_detected_at,
4967            chrono::Duration::minutes(30),
4968        );
4969
4970        let timing = stuck_task_timing_from_map(
4971            &tmux,
4972            repo,
4973            task_id,
4974            &updated_at,
4975            &config,
4976            "parse failure",
4977            &session_map,
4978        );
4979
4980        assert!(
4981            timing.is_none(),
4982            "task age should be discounted below the stuck threshold once suspend time is subtracted, got {timing:?}"
4983        );
4984
4985        super::super::suspend::clear_for_test();
4986    }
4987
4988    #[tokio::test]
4989    async fn recover_stuck_tasks_reclaims_routed_task_blocked_by_stale_session() {
4990        let store = Arc::new(TaskStore::open_memory().await.unwrap());
4991        let mock = MockBackend::new();
4992        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
4993        let task_manager = Arc::new(TaskManager::with_store(
4994            std::sync::Arc::clone(&backend),
4995            std::sync::Arc::clone(&store),
4996            "owner/repo".to_string(),
4997        ));
4998        let tmux = Arc::new(TmuxManager::new());
4999        let config = EngineConfig {
5000            no_session_stuck_timeout: 600,
5001            stuck_timeout: 1,
5002            ..EngineConfig::default()
5003        };
5004
5005        let id = store
5006            .create_internal("owner/repo", "Blocked routed task", "", "cron", "1", None)
5007            .await
5008            .unwrap();
5009        store
5010            .update_status(id, crate::store::TaskStatus::Routed)
5011            .await
5012            .unwrap();
5013        set_task_updated_at_past(&store, id).await;
5014
5015        let task_id = format!("internal:{id}");
5016        let session_name = tmux.session_name("owner/repo", &task_id);
5017        let create_result = tmux
5018            .command()
5019            .args([
5020                "new-session",
5021                "-d",
5022                "-s",
5023                &session_name,
5024                "-c",
5025                "/tmp",
5026                "sleep 60",
5027            ])
5028            .output()
5029            .await;
5030
5031        match create_result {
5032            Ok(o) if o.status.success() => {}
5033            _ => {
5034                eprintln!("Skipping test: tmux not available or failed to create test session");
5035                return;
5036            }
5037        }
5038
5039        let session_map = tmux.batch_session_active().await;
5040        assert_eq!(session_map.get(&session_name).copied(), Some(true));
5041
5042        tick_recover_stuck_tasks(
5043            &backend,
5044            &tmux,
5045            "owner/repo",
5046            &task_manager,
5047            &config,
5048            &store,
5049            &session_map,
5050            &Arc::new(DashMap::new()),
5051        )
5052        .await
5053        .unwrap();
5054
5055        let task = store.get(id).await.unwrap();
5056        assert_eq!(
5057            task.status,
5058            crate::store::TaskStatus::New,
5059            "routed task blocked by a stale session should be reset for re-routing"
5060        );
5061        assert!(
5062            !tmux.session_exists(&session_name).await,
5063            "recovery should kill the stale tmux session so dispatch can proceed"
5064        );
5065    }
5066
5067    // ── tick_dispatch_tasks: deadlock regression test (#1361) ──────────────
5068
5069    /// Regression test for #1361: tick_dispatch_tasks previously accepted
5070    /// `&Arc<RwLock<Router>>` and called `router_arc.read().await` internally.
5071    /// The main loop already held a write lock on the same RwLock, so the read
5072    /// could never be acquired — guaranteed deadlock on every tick with routed
5073    /// tasks.
5074    ///
5075    /// The fix changed the signature to accept `&Router` directly (the caller
5076    /// already has the dereferenced guard), eliminating the lock re-acquisition.
5077    ///
5078    /// This test verifies the function completes within 2 seconds when the
5079    /// caller holds the write lock — the old code would deadlock here.
5080    #[tokio::test]
5081    async fn dispatch_does_not_deadlock_under_write_lock() {
5082        use crate::engine::router::Router;
5083        use tokio::sync::RwLock;
5084
5085        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5086        let mock = MockBackend::new();
5087        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5088        let task_manager = Arc::new(TaskManager::with_store(
5089            std::sync::Arc::clone(&backend),
5090            std::sync::Arc::clone(&store),
5091            "owner/repo".to_string(),
5092        ));
5093        let tmux = Arc::new(TmuxManager::new());
5094        let semaphore = Arc::new(Semaphore::new(4));
5095        let (weight_tx, _weight_rx) = mpsc::channel(16);
5096        let dispatching = Arc::new(DashMap::new());
5097        let transport = Arc::new(Transport::new());
5098        let capture = Arc::new(CaptureService::new(transport));
5099
5100        let router_config = crate::engine::router::RouterConfig::default();
5101        let router = Router::new_for_test(router_config, vec!["claude".to_string()]);
5102        let router_arc = Arc::new(RwLock::new(router));
5103
5104        // Simulate what the main loop does: hold a write lock, snapshot
5105        // dispatch mode, then call tick_dispatch_tasks without passing any
5106        // lock guard into async dispatch work.
5107        let write_guard = router_arc.write().await;
5108        let dispatch_mode = dispatch_mode_from_router(&write_guard);
5109
5110        let result = tokio::time::timeout(
5111            std::time::Duration::from_secs(2),
5112            tick_dispatch_tasks(
5113                &backend,
5114                &tmux,
5115                "owner/repo",
5116                &Arc::new(crate::engine::runner::TaskRunner::new(
5117                    "owner/repo".to_string(),
5118                )),
5119                &capture,
5120                &semaphore,
5121                &task_manager,
5122                &weight_tx,
5123                dispatch_mode,
5124                &dispatching,
5125                &store,
5126                &std::collections::HashMap::new(),
5127            ),
5128        )
5129        .await;
5130
5131        assert!(
5132            result.is_ok(),
5133            "tick_dispatch_tasks deadlocked! It tried to acquire a read lock \
5134             on router_arc while a write lock was already held (issue #1361)"
5135        );
5136    }
5137
5138    /// Regression test for #3599: a task routed to an agent/model that entered
5139    /// cooldown *after* routing but *before* dispatch must not be dispatched —
5140    /// the stale route was decided before the failure that triggered the
5141    /// cooldown was known. Dispatch must re-check the assigned agent/model
5142    /// against the same cooldown store used at routing time and, if stale,
5143    /// leave the task in `Routed` for the next tick to re-evaluate (no PR is
5144    /// spawned, no per-task defer timer is written).
5145    #[tokio::test]
5146    async fn dispatch_skips_task_whose_routed_model_entered_cooldown() {
5147        use crate::store::{StoreRoute, TaskStatus as DbStatus, UpsertExternal};
5148
5149        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5150        let mock = MockBackend::new();
5151        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5152        let task_manager = Arc::new(TaskManager::with_store(
5153            std::sync::Arc::clone(&backend),
5154            std::sync::Arc::clone(&store),
5155            "owner/repo".to_string(),
5156        ));
5157        let tmux = Arc::new(TmuxManager::new());
5158        let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5159        let semaphore = Arc::new(Semaphore::new(4));
5160        let (weight_tx, _weight_rx) = mpsc::channel(16);
5161        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5162        let transport = Arc::new(Transport::new());
5163        let capture = Arc::new(CaptureService::new(transport));
5164
5165        let id = store
5166            .upsert_external(&UpsertExternal {
5167                repo: "owner/repo",
5168                ext_id: "999",
5169                title: "Stale routed task",
5170                body: "",
5171                author: "user",
5172                url: "https://github.com/owner/repo/issues/999",
5173                labels: &[],
5174                origin: "github",
5175            })
5176            .await
5177            .unwrap();
5178        store
5179            .store_route(&StoreRoute {
5180                id,
5181                agent: "minimax",
5182                model: Some("opus"),
5183                complexity: "medium",
5184                estimate: 0,
5185                reason: "test",
5186                profile: "{}",
5187                skills: "[]",
5188            })
5189            .await
5190            .unwrap();
5191        store.update_status(id, DbStatus::Routed).await.unwrap();
5192
5193        // Simulate a billing/rate-limit cooldown recorded for minimax:opus after
5194        // this task was routed but before it dispatched.
5195        crate::engine::cooldown::set_model_cooldown(
5196            "minimax",
5197            "opus",
5198            300,
5199            "rate_limit_usage_window",
5200        )
5201        .await;
5202
5203        let dispatch_mode = DispatchMode {
5204            is_degraded: false,
5205            healthy_agents: 1,
5206            threshold: 1,
5207        };
5208
5209        tick_dispatch_tasks(
5210            &backend,
5211            &tmux,
5212            "owner/repo",
5213            &runner,
5214            &capture,
5215            &semaphore,
5216            &task_manager,
5217            &weight_tx,
5218            dispatch_mode,
5219            &dispatching,
5220            &store,
5221            &std::collections::HashMap::new(),
5222        )
5223        .await
5224        .unwrap();
5225
5226        let task = store.get(id).await.unwrap();
5227        assert_eq!(
5228            task.status,
5229            DbStatus::Routed,
5230            "task whose routed model is now cooled down must stay Routed, not be dispatched"
5231        );
5232        assert!(
5233            dispatching.is_empty(),
5234            "dispatch guard must be released when skipping a stale-cooldown task"
5235        );
5236    }
5237
5238    #[tokio::test]
5239    async fn dispatch_skips_a_task_with_an_open_dependency() {
5240        use crate::store::{
5241            DepSource, NewDependency, StoreRoute, TaskStatus as DbStatus, UpsertExternal,
5242        };
5243
5244        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5245        let mock = MockBackend::new();
5246        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5247        let task_manager = Arc::new(TaskManager::with_store(
5248            std::sync::Arc::clone(&backend),
5249            std::sync::Arc::clone(&store),
5250            "owner/repo".to_string(),
5251        ));
5252        let tmux = Arc::new(TmuxManager::new());
5253        let runner = Arc::new(TaskRunner::new("owner/repo".to_string()));
5254        let semaphore = Arc::new(Semaphore::new(4));
5255        let (weight_tx, _weight_rx) = mpsc::channel(16);
5256        let dispatching: Arc<DashMap<String, String>> = Arc::new(DashMap::new());
5257        let transport = Arc::new(Transport::new());
5258        let capture = Arc::new(CaptureService::new(transport));
5259
5260        let id = store
5261            .upsert_external(&UpsertExternal {
5262                repo: "owner/repo",
5263                ext_id: "42",
5264                title: "Waiting routed task",
5265                body: "",
5266                author: "user",
5267                url: "https://github.com/owner/repo/issues/42",
5268                labels: &[],
5269                origin: "github",
5270            })
5271            .await
5272            .unwrap();
5273        store
5274            .replace_dependencies(
5275                id,
5276                &[NewDependency {
5277                    repo: "owner/repo".into(),
5278                    number: 7,
5279                    source: DepSource::Github,
5280                }],
5281            )
5282            .await
5283            .unwrap();
5284        store
5285            .store_route(&StoreRoute {
5286                id,
5287                agent: "claude",
5288                model: Some("haiku"),
5289                complexity: "simple",
5290                estimate: 0,
5291                reason: "test",
5292                profile: "{}",
5293                skills: "[]",
5294            })
5295            .await
5296            .unwrap();
5297        store.update_status(id, DbStatus::Routed).await.unwrap();
5298
5299        let dispatch_mode = DispatchMode {
5300            is_degraded: false,
5301            healthy_agents: 1,
5302            threshold: 1,
5303        };
5304        tick_dispatch_tasks(
5305            &backend,
5306            &tmux,
5307            "owner/repo",
5308            &runner,
5309            &capture,
5310            &semaphore,
5311            &task_manager,
5312            &weight_tx,
5313            dispatch_mode,
5314            &dispatching,
5315            &store,
5316            &std::collections::HashMap::new(),
5317        )
5318        .await
5319        .unwrap();
5320
5321        let task = store.get(id).await.unwrap();
5322        assert_eq!(task.status, DbStatus::Waiting);
5323        assert!(dispatching.is_empty());
5324    }
5325
5326    /// Regression test for the completed-session -> stuck-task reclaim race.
5327    ///
5328    /// Scenario:
5329    /// - Runner finishes a tmux session and will call store_touch_updated_at shortly after
5330    /// - Engine's tick may observe the dead session in Phase 1 and Phase 2 may run
5331    ///   stuck-task recovery before the runner touches the store, causing incorrect reclaim
5332    /// Fix: tick_check_session_completions now touches the store.updated_at when it
5333    /// observes a completed session, preventing reclaim. This test simulates the
5334    /// timing by creating a session, marking the corresponding task InProgress with
5335    /// an old updated_at, calling tick_check_session_completions (which should touch
5336    /// the store), then calling tick_recover_stuck_tasks and asserting the task
5337    /// remains InProgress.
5338    #[tokio::test]
5339    async fn completed_session_does_not_cause_stuck_reclaim_race() {
5340        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5341        let mock = MockBackend::new();
5342        let backend: Arc<dyn ExternalBackend> = Arc::new(mock);
5343        let task_manager = Arc::new(TaskManager::with_store(
5344            std::sync::Arc::clone(&backend),
5345            std::sync::Arc::clone(&store),
5346            "owner/repo".to_string(),
5347        ));
5348        let tmux = Arc::new(TmuxManager::new());
5349        let transport = Arc::new(Transport::new());
5350        let capture = Arc::new(CaptureService::new(transport));
5351
5352        // Insert external task and set to InProgress with old updated_at
5353        let id = store
5354            .upsert_external(&crate::store::UpsertExternal {
5355                repo: "owner/repo",
5356                ext_id: "200",
5357                title: "Race Task",
5358                body: "",
5359                author: "",
5360                url: "",
5361                labels: &[],
5362                origin: "github",
5363            })
5364            .await
5365            .unwrap();
5366        store
5367            .update_status(id, crate::store::TaskStatus::InProgress)
5368            .await
5369            .unwrap();
5370
5371        // set updated_at to far past so it would normally be eligible for reclaim
5372        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5373            .bind(id)
5374            .execute(store.pool())
5375            .await
5376            .unwrap();
5377
5378        // Register a real tmux session for this task and then exit it so it's observed dead
5379        let task_id = "200";
5380        let session_name = tmux.session_name("owner/repo", task_id);
5381        let create_result = tmux
5382            .command()
5383            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5384            .output()
5385            .await;
5386        match create_result {
5387            Ok(o) if o.status.success() => {}
5388            _ => {
5389                eprintln!("Skipping test: tmux not available or failed to create test session");
5390                return;
5391            }
5392        }
5393        let _guard = SessionGuard(session_name.clone());
5394
5395        // Ensure remain-on-exit so the session lingers as dead long enough to be snapshot
5396        let _ = tmux
5397            .command()
5398            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5399            .output()
5400            .await;
5401        let _ = tmux
5402            .command()
5403            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5404            .output()
5405            .await;
5406
5407        // Poll until pane is dead
5408        for _ in 0..20 {
5409            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
5410            if !tmux.session_is_running(&session_name).await {
5411                break;
5412            }
5413        }
5414
5415        // Register the session in capture (engine does this during dispatch)
5416        let task_key = task_id.to_string();
5417        capture
5418            .register_session("owner/repo", &task_key, &session_name)
5419            .await;
5420
5421        // Call phase1 (now touches store.updated_at for completed session)
5422        let config = EngineConfig {
5423            no_session_stuck_timeout: 600,
5424            stuck_timeout: 1800,
5425            ..EngineConfig::default()
5426        };
5427
5428        tick_check_session_completions(&tmux, "owner/repo", &capture, &store)
5429            .await
5430            .unwrap();
5431
5432        // Verify phase1 actually touched updated_at. It only processes sessions
5433        // that appear in snapshot() (list_sessions + batch_session_active). On CI,
5434        // if the session was already fully removed before the snapshot runs (e.g.
5435        // remain-on-exit semantics differ across tmux versions), phase1 won't see
5436        // it and the store won't be touched. Skip in that case.
5437        {
5438            let task_state = store.get(id).await.unwrap();
5439            if task_state.updated_at.starts_with("2020") {
5440                eprintln!(
5441                    "Skipping test: tick_check_session_completions did not update stored \
5442                     updated_at — session was not in snapshot (CI tmux environment issue)"
5443                );
5444                return;
5445            }
5446        }
5447
5448        // Now run phase2 — the updated_at should have been touched so no reclaim occurs
5449        tick_recover_stuck_tasks(
5450            &backend,
5451            &tmux,
5452            "owner/repo",
5453            &task_manager,
5454            &config,
5455            &store,
5456            &std::collections::HashMap::new(),
5457            &Arc::new(DashMap::new()),
5458        )
5459        .await
5460        .unwrap();
5461
5462        // Verify the task is still InProgress in the store
5463        let task = store.get(id).await.unwrap();
5464        assert_eq!(
5465            task.status,
5466            crate::store::TaskStatus::InProgress,
5467            "task should not be reclaimed to New"
5468        );
5469    }
5470
5471    /// Regression test: a repo configured with a ".git" suffix (e.g. "owner/repo.git")
5472    /// must still match its own live tmux sessions in `tick_check_session_completions`.
5473    ///
5474    /// `TmuxManager::session_name` strips ".git" when creating the session, so a live
5475    /// session's parsed `project` field never carries it. Before the fix, this
5476    /// function's `repo_name` derivation skipped the `.trim_end_matches(".git")` step,
5477    /// so `session.project` ("repo") never equaled `repo_name` ("repo.git") and the
5478    /// completed-session touch/kill was silently skipped for the entire repo — leaving
5479    /// the coarser-grained stuck-task reclaim as the only (racy) safety net.
5480    #[tokio::test]
5481    async fn session_completion_matches_repo_configured_with_git_suffix() {
5482        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5483        let tmux = Arc::new(TmuxManager::new());
5484        let transport = Arc::new(Transport::new());
5485        let capture = Arc::new(CaptureService::new(transport));
5486
5487        let repo = "owner/repo.git";
5488        let id = store
5489            .upsert_external(&crate::store::UpsertExternal {
5490                repo,
5491                ext_id: "201",
5492                title: "Git-suffix repo task",
5493                body: "",
5494                author: "",
5495                url: "",
5496                labels: &[],
5497                origin: "github",
5498            })
5499            .await
5500            .unwrap();
5501        store
5502            .update_status(id, crate::store::TaskStatus::InProgress)
5503            .await
5504            .unwrap();
5505        sqlx::query("UPDATE tasks SET updated_at = '2020-01-01T00:00:00Z' WHERE id = ?")
5506            .bind(id)
5507            .execute(store.pool())
5508            .await
5509            .unwrap();
5510
5511        // session_name strips ".git" internally — this is the real session a dispatch
5512        // for this repo would have created.
5513        let task_id = "201";
5514        let session_name = tmux.session_name(repo, task_id);
5515        let create_result = tmux
5516            .command()
5517            .args(["new-session", "-d", "-s", &session_name, "-c", "/tmp"])
5518            .output()
5519            .await;
5520        match create_result {
5521            Ok(o) if o.status.success() => {}
5522            _ => {
5523                eprintln!("Skipping test: tmux not available or failed to create test session");
5524                return;
5525            }
5526        }
5527        let _guard = SessionGuard(session_name.clone());
5528
5529        let _ = tmux
5530            .command()
5531            .args(["set-option", "-t", &session_name, "remain-on-exit", "on"])
5532            .output()
5533            .await;
5534        let _ = tmux
5535            .command()
5536            .args(["send-keys", "-t", &session_name, "exit", "Enter"])
5537            .output()
5538            .await;
5539
5540        for _ in 0..20 {
5541            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
5542            if !tmux.session_is_running(&session_name).await {
5543                break;
5544            }
5545        }
5546
5547        capture.register_session(repo, task_id, &session_name).await;
5548
5549        tick_check_session_completions(&tmux, repo, &capture, &store)
5550            .await
5551            .unwrap();
5552
5553        let task_state = store.get(id).await.unwrap();
5554        if task_state.updated_at.starts_with("2020") {
5555            eprintln!(
5556                "Skipping test: tick_check_session_completions did not update stored \
5557                 updated_at — session was not in snapshot (CI tmux environment issue)"
5558            );
5559            return;
5560        }
5561        assert!(
5562            !task_state.updated_at.starts_with("2020"),
5563            "tick_check_session_completions must match its own repo's live sessions \
5564             even when the repo is configured with a \".git\" suffix"
5565        );
5566    }
5567
5568    // ── tick_job_scheduler: resilience to bad project configs ────────────────
5569
5570    /// A project with a duplicate job id (same id in .orch.yml AND prompts/jobs/)
5571    /// must not cause tick_job_scheduler to return Err — it should log an error
5572    /// and return Ok(()) so other projects are unaffected.
5573    #[tokio::test]
5574    async fn job_scheduler_returns_ok_on_duplicate_job_config() {
5575        let tmp = tempfile::tempdir().unwrap();
5576        let cfg = tmp.path().join(".orch.yml");
5577        std::fs::write(
5578            &cfg,
5579            "jobs:\n  - id: trading-scan\n    schedule: \"0 9 * * *\"\n    task:\n      title: Scan\n      body: do it\n",
5580        )
5581        .unwrap();
5582        let jobs_dir = tmp.path().join("prompts").join("jobs");
5583        std::fs::create_dir_all(&jobs_dir).unwrap();
5584        std::fs::write(
5585            jobs_dir.join("trading-scan.md"),
5586            "---\nid: trading-scan\nschedule: '0 9 * * *'\ntitle: Trading Scan\n---\n\nDo trading scan.\n",
5587        )
5588        .unwrap();
5589
5590        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5591        let result = tick_job_scheduler(&cfg, &backend, None, "test/repo", None).await;
5592        assert!(
5593            result.is_ok(),
5594            "duplicate job id must not propagate as Err: {result:?}"
5595        );
5596    }
5597
5598    /// When one project has a duplicate job id (bad config) and a second project
5599    /// has a valid config, the scheduler tick for the second project must still
5600    /// succeed — simulating the per-project loop in the engine.
5601    #[tokio::test]
5602    async fn job_scheduler_processes_valid_project_when_other_has_duplicate() {
5603        // Bad project: duplicate id across .orch.yml and prompts/jobs/
5604        let bad_tmp = tempfile::tempdir().unwrap();
5605        let bad_cfg = bad_tmp.path().join(".orch.yml");
5606        std::fs::write(
5607            &bad_cfg,
5608            "jobs:\n  - id: dup\n    schedule: \"0 9 * * *\"\n    task:\n      title: Dup\n      body: x\n",
5609        )
5610        .unwrap();
5611        let bad_jobs_dir = bad_tmp.path().join("prompts").join("jobs");
5612        std::fs::create_dir_all(&bad_jobs_dir).unwrap();
5613        std::fs::write(
5614            bad_jobs_dir.join("dup.md"),
5615            "---\nid: dup\nschedule: '0 9 * * *'\ntitle: Dup\n---\n\nbody\n",
5616        )
5617        .unwrap();
5618
5619        // Good project: valid config, no duplicate
5620        let good_tmp = tempfile::tempdir().unwrap();
5621        let good_cfg = good_tmp.path().join(".orch.yml");
5622        std::fs::write(
5623            &good_cfg,
5624            "jobs:\n  - id: valid-job\n    schedule: \"0 9 * * *\"\n    task:\n      title: Valid\n      body: ok\n",
5625        )
5626        .unwrap();
5627
5628        let backend: Arc<dyn ExternalBackend> = Arc::new(MockBackend::new());
5629
5630        let bad_result = tick_job_scheduler(&bad_cfg, &backend, None, "test/bad-repo", None).await;
5631        assert!(
5632            bad_result.is_ok(),
5633            "bad project must not abort scheduler: {bad_result:?}"
5634        );
5635
5636        let good_result =
5637            tick_job_scheduler(&good_cfg, &backend, None, "test/good-repo", None).await;
5638        assert!(
5639            good_result.is_ok(),
5640            "valid project must still run after bad project: {good_result:?}"
5641        );
5642    }
5643
5644    // ── route_new_tasks_global ───────────────────────────────────────────────
5645
5646    /// Regression test for #3407: tasks from inactive/removed repos were stuck
5647    /// in `new` indefinitely because `tick_route_tasks` is repo-scoped.
5648    ///
5649    /// `route_new_tasks_global` must route tasks whose repo != current_repo
5650    /// and leave current-repo tasks untouched.
5651    #[serial_test::serial(cooldown_state)]
5652    #[tokio::test]
5653    async fn global_sweep_routes_inactive_repo_tasks() {
5654        use crate::engine::router::{Router, RouterConfig};
5655        use crate::store::{TaskStatus, UpsertExternal};
5656
5657        crate::engine::cooldown::reset_global_state().await;
5658
5659        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5660
5661        // Active repo task — must NOT be touched by the global sweep.
5662        let active_id = store
5663            .upsert_external(&UpsertExternal {
5664                repo: "active/repo",
5665                ext_id: "1",
5666                title: "Active task",
5667                body: "",
5668                author: "user",
5669                url: "https://github.com/active/repo/issues/1",
5670                labels: &[],
5671                origin: "github",
5672            })
5673            .await
5674            .unwrap();
5675
5676        // Inactive repo task — must be routed by the global sweep.
5677        let inactive_id = store
5678            .upsert_external(&UpsertExternal {
5679                repo: "inactive/repo",
5680                ext_id: "42",
5681                title: "Orphaned task",
5682                body: "",
5683                author: "user",
5684                url: "https://github.com/inactive/repo/issues/42",
5685                labels: &[],
5686                origin: "github",
5687            })
5688            .await
5689            .unwrap();
5690
5691        assert_eq!(store.get(active_id).await.unwrap().status, TaskStatus::New);
5692        assert_eq!(
5693            store.get(inactive_id).await.unwrap().status,
5694            TaskStatus::New
5695        );
5696
5697        let mut model_map: std::collections::HashMap<
5698            String,
5699            std::collections::HashMap<String, Vec<String>>,
5700        > = std::collections::HashMap::new();
5701        let mut tier = std::collections::HashMap::new();
5702        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5703        model_map.insert("medium".to_string(), tier);
5704        let config = RouterConfig {
5705            mode: "round_robin".to_string(),
5706            agents: vec!["claude".to_string()],
5707            model_map,
5708            ..Default::default()
5709        };
5710        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5711
5712        let active_repos: std::collections::HashSet<String> =
5713            ["active/repo".to_string()].into_iter().collect();
5714        let mut routing_quota = RoutingTickQuota::new(1);
5715        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5716            .await
5717            .unwrap();
5718
5719        // Active-repo task must remain new (global sweep skips it).
5720        assert_eq!(
5721            store.get(active_id).await.unwrap().status,
5722            TaskStatus::New,
5723            "global sweep must not touch tasks for the active repo"
5724        );
5725
5726        // Inactive-repo task must now be routed.
5727        let after = store.get(inactive_id).await.unwrap();
5728        assert_eq!(
5729            after.status,
5730            TaskStatus::Routed,
5731            "global sweep must route tasks from inactive repos"
5732        );
5733        assert!(
5734            matches!(&after.agent, Some(a) if !a.is_empty()),
5735            "agent must be set after routing"
5736        );
5737    }
5738
5739    /// Regression test for #3523: `route_new_tasks_global` previously filtered only
5740    /// `t.repo != current_repo`, so during repo A's tick it would route repo B's `new`
5741    /// tasks too — even though B is an active, non-orphaned project with its own tick
5742    /// scheduled later in the same pass. This is the exact "racing the real per-repo
5743    /// dispatch" failure mode `dispatch_routed_tasks_global` was fixed for in #3415;
5744    /// the route-side sweep needs the same `active_repos` scoping, not just the
5745    /// current tick's own repo.
5746    #[serial_test::serial(cooldown_state)]
5747    #[tokio::test]
5748    async fn global_sweep_does_not_touch_other_active_repo_tasks() {
5749        use crate::engine::router::{Router, RouterConfig};
5750        use crate::store::{TaskStatus, UpsertExternal};
5751
5752        crate::engine::cooldown::reset_global_state().await;
5753
5754        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5755
5756        // Belongs to a second active repo — NOT the repo whose tick is running,
5757        // but still a currently-configured project, not an orphan.
5758        let other_active_id = store
5759            .upsert_external(&UpsertExternal {
5760                repo: "other-active/repo",
5761                ext_id: "7",
5762                title: "Other active project's task",
5763                body: "",
5764                author: "user",
5765                url: "https://github.com/other-active/repo/issues/7",
5766                labels: &[],
5767                origin: "github",
5768            })
5769            .await
5770            .unwrap();
5771
5772        // Genuinely orphaned repo — not in the active set at all.
5773        let inactive_id = store
5774            .upsert_external(&UpsertExternal {
5775                repo: "inactive/repo",
5776                ext_id: "42",
5777                title: "Orphaned task",
5778                body: "",
5779                author: "user",
5780                url: "https://github.com/inactive/repo/issues/42",
5781                labels: &[],
5782                origin: "github",
5783            })
5784            .await
5785            .unwrap();
5786
5787        let mut model_map: std::collections::HashMap<
5788            String,
5789            std::collections::HashMap<String, Vec<String>>,
5790        > = std::collections::HashMap::new();
5791        let mut tier = std::collections::HashMap::new();
5792        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5793        model_map.insert("medium".to_string(), tier);
5794        let config = RouterConfig {
5795            mode: "round_robin".to_string(),
5796            agents: vec!["claude".to_string()],
5797            model_map,
5798            ..Default::default()
5799        };
5800        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5801
5802        // Simulate "active/repo"'s tick running the global sweep. Both
5803        // "active/repo" and "other-active/repo" are currently-configured
5804        // projects; only "inactive/repo" is orphaned.
5805        let active_repos: std::collections::HashSet<String> =
5806            ["active/repo".to_string(), "other-active/repo".to_string()]
5807                .into_iter()
5808                .collect();
5809        let mut routing_quota = RoutingTickQuota::new(1);
5810        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5811            .await
5812            .unwrap();
5813
5814        assert_eq!(
5815            store.get(other_active_id).await.unwrap().status,
5816            TaskStatus::New,
5817            "global sweep run from a different active repo's tick must not route \
5818             another active repo's new tasks — that repo's own tick will route them"
5819        );
5820        assert_eq!(
5821            store.get(inactive_id).await.unwrap().status,
5822            TaskStatus::Routed,
5823            "global sweep must still route tasks from genuinely orphaned repos"
5824        );
5825    }
5826
5827    /// Tasks with the `no-agent` label must be skipped by the global sweep.
5828    #[serial_test::serial(cooldown_state)]
5829    #[tokio::test]
5830    async fn global_sweep_skips_no_agent_tasks() {
5831        use crate::engine::router::{Router, RouterConfig};
5832        use crate::store::{TaskStatus, UpsertExternal};
5833
5834        crate::engine::cooldown::reset_global_state().await;
5835
5836        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5837
5838        let id = store
5839            .upsert_external(&UpsertExternal {
5840                repo: "inactive/repo",
5841                ext_id: "99",
5842                title: "No-agent task",
5843                body: "",
5844                author: "user",
5845                url: "https://github.com/inactive/repo/issues/99",
5846                labels: &["no-agent".to_string()],
5847                origin: "github",
5848            })
5849            .await
5850            .unwrap();
5851
5852        let mut model_map: std::collections::HashMap<
5853            String,
5854            std::collections::HashMap<String, Vec<String>>,
5855        > = std::collections::HashMap::new();
5856        let mut tier = std::collections::HashMap::new();
5857        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5858        model_map.insert("medium".to_string(), tier);
5859        let config = RouterConfig {
5860            mode: "round_robin".to_string(),
5861            agents: vec!["claude".to_string()],
5862            model_map,
5863            ..Default::default()
5864        };
5865        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5866
5867        let active_repos: std::collections::HashSet<String> =
5868            ["active/repo".to_string()].into_iter().collect();
5869        let mut routing_quota = RoutingTickQuota::new(1);
5870        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5871            .await
5872            .unwrap();
5873
5874        assert_eq!(
5875            store.get(id).await.unwrap().status,
5876            TaskStatus::New,
5877            "no-agent tasks must not be routed"
5878        );
5879    }
5880
5881    /// Regression test for #3615: the routing limit must be consumed by a
5882    /// shared per-engine-tick quota, not independently by every repo/global
5883    /// routing pass. Otherwise one slow router timeout per pass can stack into
5884    /// a multi-minute watchdog stall.
5885    #[serial_test::serial(cooldown_state)]
5886    #[tokio::test]
5887    async fn global_sweep_honors_shared_routing_quota() {
5888        use crate::engine::router::{Router, RouterConfig};
5889        use crate::store::{TaskStatus, UpsertExternal};
5890
5891        crate::engine::cooldown::reset_global_state().await;
5892
5893        let store = Arc::new(TaskStore::open_memory().await.unwrap());
5894        for ext_id in ["1", "2"] {
5895            store
5896                .upsert_external(&UpsertExternal {
5897                    repo: "inactive/repo",
5898                    ext_id,
5899                    title: "Orphaned task",
5900                    body: "",
5901                    author: "user",
5902                    url: &format!("https://github.com/inactive/repo/issues/{ext_id}"),
5903                    labels: &[],
5904                    origin: "github",
5905                })
5906                .await
5907                .unwrap();
5908        }
5909
5910        let mut model_map: std::collections::HashMap<
5911            String,
5912            std::collections::HashMap<String, Vec<String>>,
5913        > = std::collections::HashMap::new();
5914        let mut tier = std::collections::HashMap::new();
5915        tier.insert("claude".to_string(), vec!["haiku".to_string()]);
5916        model_map.insert("medium".to_string(), tier);
5917        let config = RouterConfig {
5918            mode: "round_robin".to_string(),
5919            agents: vec!["claude".to_string()],
5920            model_map,
5921            ..Default::default()
5922        };
5923        let mut router = Router::new_for_test(config, vec!["claude".to_string()]);
5924
5925        let active_repos: std::collections::HashSet<String> =
5926            ["active/repo".to_string()].into_iter().collect();
5927        let mut routing_quota = RoutingTickQuota::new(1);
5928        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5929            .await
5930            .unwrap();
5931
5932        let all_tasks = store.list_all_global().await.unwrap();
5933        let routed = all_tasks
5934            .iter()
5935            .filter(|task| task.status == TaskStatus::Routed)
5936            .count();
5937        let still_new = all_tasks
5938            .iter()
5939            .filter(|task| task.status == TaskStatus::New)
5940            .count();
5941
5942        assert_eq!(routed, 1, "only one task may consume the shared quota");
5943        assert_eq!(still_new, 1, "remaining tasks wait for the next tick");
5944
5945        route_new_tasks_global(&mut router, &store, &active_repos, &mut routing_quota)
5946            .await
5947            .unwrap();
5948        let after_exhausted = store.list_all_global().await.unwrap();
5949        assert_eq!(
5950            after_exhausted
5951                .iter()
5952                .filter(|task| task.status == TaskStatus::Routed)
5953                .count(),
5954            1,
5955            "an exhausted shared quota must not reset within the same tick"
5956        );
5957    }
5958
5959    // ── dispatch_routed_tasks_global ─────────────────────────────────────────
5960    //
5961    // `dispatch_routed_tasks_global` itself is deliberately not exercised
5962    // end-to-end here: once it finds an orphaned repo, it spawns the real
5963    // runner, which creates a real tmux session and touches the network.
5964    // The existing `dispatch_does_not_deadlock_under_write_lock` test avoids
5965    // this same hazard by keeping the dispatchable set empty. The selection
5966    // logic (which tasks belong to which repo) is pulled into the pure
5967    // `group_orphaned_routed_tasks` helper below so it can be tested without
5968    // those side effects.
5969
5970    /// Regression test for #3413: tasks routed for an inactive/removed repo were
5971    /// stuck in `routed` indefinitely because `tick_dispatch_tasks` is repo-scoped.
5972    ///
5973    /// `group_orphaned_routed_tasks` must select tasks whose repo != the active
5974    /// repo, group them by repo, and leave active-repo tasks out entirely.
5975    #[tokio::test]
5976    async fn group_orphaned_routed_tasks_selects_inactive_repo_tasks() {
5977        use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
5978
5979        async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
5980            let id = store
5981                .upsert_external(&UpsertExternal {
5982                    repo,
5983                    ext_id,
5984                    title: "Routed task",
5985                    body: "",
5986                    author: "user",
5987                    url: &format!("https://github.com/{repo}/issues/{ext_id}"),
5988                    labels: &[],
5989                    origin: "github",
5990                })
5991                .await
5992                .unwrap();
5993            store
5994                .store_route(&StoreRoute {
5995                    id,
5996                    agent: "claude",
5997                    model: Some("sonnet"),
5998                    complexity: "medium",
5999                    estimate: 0,
6000                    reason: "test",
6001                    profile: "{}",
6002                    skills: "[]",
6003                })
6004                .await
6005                .unwrap();
6006            store.update_status(id, TaskStatus::Routed).await.unwrap();
6007            id
6008        }
6009
6010        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6011
6012        let active_id = make_routed(&store, "active/repo", "1").await;
6013        let inactive_id_a = make_routed(&store, "inactive/repo", "2").await;
6014        let other_inactive_id = make_routed(&store, "other-inactive/repo", "3").await;
6015        let inactive_id_b = make_routed(&store, "inactive/repo", "4").await;
6016
6017        let all_routed = store
6018            .list_all_by_status_global(TaskStatus::Routed)
6019            .await
6020            .unwrap();
6021        let active_repos: std::collections::HashSet<String> =
6022            ["active/repo".to_string()].into_iter().collect();
6023        let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6024
6025        assert!(
6026            !by_repo.contains_key("active/repo"),
6027            "global sweep must not select tasks for the active repo"
6028        );
6029        let inactive_ids: std::collections::HashSet<i64> = by_repo
6030            .get("inactive/repo")
6031            .expect("inactive/repo must be selected")
6032            .iter()
6033            .map(|t| t.id)
6034            .collect();
6035        assert_eq!(
6036            inactive_ids,
6037            [inactive_id_a, inactive_id_b].into_iter().collect(),
6038            "global sweep must group all orphaned tasks for a repo together"
6039        );
6040        assert_eq!(
6041            by_repo
6042                .get("other-inactive/repo")
6043                .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6044            Some(vec![other_inactive_id])
6045        );
6046
6047        // Sanity: the active-repo task really was created and routed, it's just
6048        // excluded from the selection above.
6049        assert_eq!(
6050            store.get(active_id).await.unwrap().status,
6051            TaskStatus::Routed
6052        );
6053    }
6054
6055    /// Regression test for the PR #3413 review finding: a single `current_repo` string is
6056    /// the wrong exclusion — repo B's routed tasks must not look "orphaned" during repo A's
6057    /// dispatch just because A != B.
6058    /// `group_orphaned_routed_tasks` must exclude every repo in the full active set.
6059    #[tokio::test]
6060    async fn group_orphaned_routed_tasks_excludes_all_active_repos_not_just_current() {
6061        use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
6062
6063        async fn make_routed(store: &TaskStore, repo: &str, ext_id: &str) -> i64 {
6064            let id = store
6065                .upsert_external(&UpsertExternal {
6066                    repo,
6067                    ext_id,
6068                    title: "Routed task",
6069                    body: "",
6070                    author: "user",
6071                    url: &format!("https://github.com/{repo}/issues/{ext_id}"),
6072                    labels: &[],
6073                    origin: "github",
6074                })
6075                .await
6076                .unwrap();
6077            store
6078                .store_route(&StoreRoute {
6079                    id,
6080                    agent: "claude",
6081                    model: Some("sonnet"),
6082                    complexity: "medium",
6083                    estimate: 0,
6084                    reason: "test",
6085                    profile: "{}",
6086                    skills: "[]",
6087                })
6088                .await
6089                .unwrap();
6090            store.update_status(id, TaskStatus::Routed).await.unwrap();
6091            id
6092        }
6093
6094        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6095
6096        let repo_a_id = make_routed(&store, "active/repo-a", "1").await;
6097        let repo_b_id = make_routed(&store, "active/repo-b", "2").await;
6098        let orphaned_id = make_routed(&store, "inactive/repo", "3").await;
6099
6100        let all_routed = store
6101            .list_all_by_status_global(TaskStatus::Routed)
6102            .await
6103            .unwrap();
6104        // Simulate repo A's tick: current repo is "active/repo-a", but the full
6105        // active project set also includes "active/repo-b".
6106        let active_repos: std::collections::HashSet<String> =
6107            ["active/repo-a".to_string(), "active/repo-b".to_string()]
6108                .into_iter()
6109                .collect();
6110        let by_repo = group_orphaned_routed_tasks(all_routed, &active_repos);
6111
6112        assert!(
6113            !by_repo.contains_key("active/repo-a"),
6114            "the current repo's own tasks must not be swept"
6115        );
6116        assert!(
6117            !by_repo.contains_key("active/repo-b"),
6118            "another active repo's tasks must not be swept during repo A's tick"
6119        );
6120        assert_eq!(
6121            by_repo
6122                .get("inactive/repo")
6123                .map(|tasks| tasks.iter().map(|t| t.id).collect::<Vec<_>>()),
6124            Some(vec![orphaned_id]),
6125            "a genuinely inactive repo's tasks must still be swept"
6126        );
6127
6128        // Sanity: repo A and repo B tasks were really created and routed, just excluded.
6129        assert_eq!(
6130            store.get(repo_a_id).await.unwrap().status,
6131            TaskStatus::Routed
6132        );
6133        assert_eq!(
6134            store.get(repo_b_id).await.unwrap().status,
6135            TaskStatus::Routed
6136        );
6137    }
6138
6139    /// Tasks with the `no-agent` label must be skipped by the global dispatch sweep.
6140    #[tokio::test]
6141    async fn global_dispatch_sweep_skips_no_agent_tasks() {
6142        use crate::store::{StoreRoute, TaskStatus, UpsertExternal};
6143
6144        let store = Arc::new(TaskStore::open_memory().await.unwrap());
6145
6146        let id = store
6147            .upsert_external(&UpsertExternal {
6148                repo: "inactive/repo",
6149                ext_id: "99",
6150                title: "No-agent routed task",
6151                body: "",
6152                author: "user",
6153                url: "https://github.com/inactive/repo/issues/99",
6154                labels: &["no-agent".to_string()],
6155                origin: "github",
6156            })
6157            .await
6158            .unwrap();
6159        store
6160            .store_route(&StoreRoute {
6161                id,
6162                agent: "claude",
6163                model: Some("sonnet"),
6164                complexity: "medium",
6165                estimate: 0,
6166                reason: "test",
6167                profile: "{}",
6168                skills: "[]",
6169            })
6170            .await
6171            .unwrap();
6172        store.update_status(id, TaskStatus::Routed).await.unwrap();
6173
6174        // Safe to call the full sweep here: with the only orphaned task filtered
6175        // out by the no-agent label, `group_orphaned_routed_tasks` returns an
6176        // empty map and `dispatch_routed_tasks_global` returns before ever
6177        // constructing a backend or spawning a runner.
6178        let tmux = Arc::new(TmuxManager::new());
6179        let transport = Arc::new(Transport::new());
6180        let capture = Arc::new(CaptureService::new(transport));
6181        let semaphore = Arc::new(Semaphore::new(4));
6182        let (weight_tx, _weight_rx) = mpsc::channel(16);
6183        let dispatching = Arc::new(DashMap::new());
6184        let dispatch_mode = DispatchMode {
6185            is_degraded: false,
6186            healthy_agents: 1,
6187            threshold: 1,
6188        };
6189
6190        let active_repos: std::collections::HashSet<String> =
6191            ["active/repo".to_string()].into_iter().collect();
6192        dispatch_routed_tasks_global(
6193            &tmux,
6194            &capture,
6195            &semaphore,
6196            &weight_tx,
6197            dispatch_mode,
6198            &dispatching,
6199            &store,
6200            &std::collections::HashMap::new(),
6201            &active_repos,
6202        )
6203        .await
6204        .unwrap();
6205
6206        assert_eq!(
6207            store.get(id).await.unwrap().status,
6208            TaskStatus::Routed,
6209            "no-agent tasks must not be dispatched"
6210        );
6211    }
6212}